LLVM 24.0.0git
ARMISelLowering.cpp
Go to the documentation of this file.
1//===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that ARM uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARMISelLowering.h"
15#include "ARMBaseInstrInfo.h"
16#include "ARMBaseRegisterInfo.h"
17#include "ARMCallingConv.h"
20#include "ARMPerfectShuffle.h"
21#include "ARMRegisterInfo.h"
22#include "ARMSelectionDAGInfo.h"
23#include "ARMSubtarget.h"
27#include "Utils/ARMBaseInfo.h"
28#include "llvm/ADT/APFloat.h"
29#include "llvm/ADT/APInt.h"
30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/BitVector.h"
32#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/Statistic.h"
38#include "llvm/ADT/StringRef.h"
40#include "llvm/ADT/Twine.h"
67#include "llvm/IR/Attributes.h"
68#include "llvm/IR/CallingConv.h"
69#include "llvm/IR/Constant.h"
70#include "llvm/IR/Constants.h"
71#include "llvm/IR/DataLayout.h"
72#include "llvm/IR/DebugLoc.h"
74#include "llvm/IR/Function.h"
75#include "llvm/IR/GlobalAlias.h"
76#include "llvm/IR/GlobalValue.h"
78#include "llvm/IR/IRBuilder.h"
79#include "llvm/IR/InlineAsm.h"
80#include "llvm/IR/Instruction.h"
83#include "llvm/IR/Intrinsics.h"
84#include "llvm/IR/IntrinsicsARM.h"
85#include "llvm/IR/Module.h"
86#include "llvm/IR/Type.h"
87#include "llvm/IR/User.h"
88#include "llvm/IR/Value.h"
89#include "llvm/MC/MCInstrDesc.h"
91#include "llvm/MC/MCSchedule.h"
98#include "llvm/Support/Debug.h"
106#include <algorithm>
107#include <cassert>
108#include <cstdint>
109#include <iterator>
110#include <limits>
111#include <optional>
112#include <tuple>
113#include <utility>
114#include <vector>
115
116using namespace llvm;
117using namespace llvm::SDPatternMatch;
118
119#define DEBUG_TYPE "arm-isel"
120
121STATISTIC(NumTailCalls, "Number of tail calls");
122STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
123STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
124STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
125STATISTIC(NumConstpoolPromoted,
126 "Number of constants with their storage promoted into constant pools");
127
128static cl::opt<bool>
129ARMInterworking("arm-interworking", cl::Hidden,
130 cl::desc("Enable / disable ARM interworking (for debugging only)"),
131 cl::init(true));
132
134 "arm-promote-constant", cl::Hidden,
135 cl::desc("Enable / disable promotion of unnamed_addr constants into "
136 "constant pools"),
137 cl::init(false)); // FIXME: set to true by default once PR32780 is fixed
139 "arm-promote-constant-max-size", cl::Hidden,
140 cl::desc("Maximum size of constant to promote into a constant pool"),
141 cl::init(64));
143 "arm-promote-constant-max-total", cl::Hidden,
144 cl::desc("Maximum size of ALL constants to promote into a constant pool"),
145 cl::init(128));
146
148MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden,
149 cl::desc("Maximum interleave factor for MVE VLDn to generate."),
150 cl::init(2));
151
153 "arm-max-base-updates-to-check", cl::Hidden,
154 cl::desc("Maximum number of base-updates to check generating postindex."),
155 cl::init(64));
156
157/// Value type used for "flags" operands / results (either CPSR or FPSCR_NZCV).
158constexpr MVT FlagsVT = MVT::i32;
159
160// The APCS parameter registers.
161static const MCPhysReg GPRArgRegs[] = {
162 ARM::R0, ARM::R1, ARM::R2, ARM::R3
163};
164
166 SelectionDAG &DAG, const SDLoc &DL) {
168 assert(Arg.ArgVT.bitsLT(MVT::i32));
169 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, Arg.ArgVT, Value);
170 SDValue Ext =
172 MVT::i32, Trunc);
173 return Ext;
174}
175
176void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) {
177 if (VT != PromotedLdStVT) {
179 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT);
180
182 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT);
183 }
184
185 MVT ElemTy = VT.getVectorElementType();
186 if (ElemTy != MVT::f64)
190 if (ElemTy == MVT::i32) {
195 } else {
200 }
209 if (VT.isInteger()) {
213 }
214
215 // Neon does not support vector divide/remainder operations.
224
225 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
226 for (auto Opcode : {ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::SMIN, ISD::SMAX,
228 setOperationAction(Opcode, VT, Legal);
229 if (!VT.isFloatingPoint())
230 for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
231 setOperationAction(Opcode, VT, Legal);
232}
233
234void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
235 addRegisterClass(VT, &ARM::DPRRegClass);
236 addTypeForNEON(VT, MVT::f64);
237}
238
239void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
240 addRegisterClass(VT, &ARM::DPairRegClass);
241 addTypeForNEON(VT, MVT::v2f64);
242}
243
244void ARMTargetLowering::setAllExpand(MVT VT) {
245 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
247
248 // We support these really simple operations even on types where all
249 // the actual arithmetic has to be broken down into simpler
250 // operations or turned into library calls.
255}
256
257void ARMTargetLowering::addAllExtLoads(const MVT From, const MVT To,
258 LegalizeAction Action) {
259 setLoadExtAction(ISD::EXTLOAD, From, To, Action);
260 setLoadExtAction(ISD::ZEXTLOAD, From, To, Action);
261 setLoadExtAction(ISD::SEXTLOAD, From, To, Action);
262}
263
264void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
265 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
266
267 for (auto VT : IntTypes) {
268 addRegisterClass(VT, &ARM::MQPRRegClass);
299
300 // No native support for these.
310
311 // Vector reductions
321
322 if (!HasMVEFP) {
327 } else {
330 }
331
332 // Pre and Post inc are supported on loads and stores
333 for (unsigned im = (unsigned)ISD::PRE_INC;
334 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
339 }
340 }
341
342 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
343 for (auto VT : FloatTypes) {
344 addRegisterClass(VT, &ARM::MQPRRegClass);
345 if (!HasMVEFP)
346 setAllExpand(VT);
347
348 // These are legal or custom whether we have MVE.fp or not
361
362 // Pre and Post inc are supported on loads and stores
363 for (unsigned im = (unsigned)ISD::PRE_INC;
364 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
369 }
370
371 if (HasMVEFP) {
374 for (auto Op :
379 }
384
385 // No native support for these.
400 }
401 }
402
403 // Custom Expand smaller than legal vector reductions to prevent false zero
404 // items being added.
413
414 // We 'support' these types up to bitcast/load/store level, regardless of
415 // MVE integer-only / float support. Only doing FP data processing on the FP
416 // vector types is inhibited at integer-only level.
417 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
418 for (auto VT : LongTypes) {
419 addRegisterClass(VT, &ARM::MQPRRegClass);
420 setAllExpand(VT);
426 }
428
429 // We can do bitwise operations on v2i64 vectors
430 setOperationAction(ISD::AND, MVT::v2i64, Legal);
431 setOperationAction(ISD::OR, MVT::v2i64, Legal);
432 setOperationAction(ISD::XOR, MVT::v2i64, Legal);
433
434 // It is legal to extload from v4i8 to v4i16 or v4i32.
435 addAllExtLoads(MVT::v8i16, MVT::v8i8, Legal);
436 addAllExtLoads(MVT::v4i32, MVT::v4i16, Legal);
437 addAllExtLoads(MVT::v4i32, MVT::v4i8, Legal);
438
439 // It is legal to sign extend from v4i8/v4i16 to v4i32 or v8i8 to v8i16.
445
446 // Some truncating stores are legal too.
447 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
448 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
449 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
450
451 // Pre and Post inc on these are legal, given the correct extends
452 for (unsigned im = (unsigned)ISD::PRE_INC;
453 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
454 for (auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
459 }
460 }
461
462 // Predicate types
463 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
464 for (auto VT : pTypes) {
465 addRegisterClass(VT, &ARM::VCCRRegClass);
480
481 if (!HasMVEFP) {
486 }
487 }
491 setOperationAction(ISD::OR, MVT::v2i1, Expand);
497
506}
507
508static bool hasNativeAtomics(const ARMSubtarget &ST) {
509 return ST.hasAnyDataBarrier() && (!ST.isThumb() || ST.hasV8MBaselineOps());
510}
511
513 return static_cast<const ARMBaseTargetMachine &>(getTargetMachine());
514}
515
517 const ARMSubtarget &STI)
518 : TargetLowering(TM_, STI), Subtarget(&STI),
519 RegInfo(Subtarget->getRegisterInfo()),
520 Itins(Subtarget->getInstrItineraryData()) {
521 const auto &TM = static_cast<const ARMBaseTargetMachine &>(TM_);
522
525
526 const Triple &TT = TM.getTargetTriple();
527
528 if (Subtarget->isThumb1Only())
529 addRegisterClass(MVT::i32, &ARM::tGPRRegClass);
530 else
531 addRegisterClass(MVT::i32, &ARM::GPRRegClass);
532
533 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() &&
534 Subtarget->hasFPRegs()) {
535 addRegisterClass(MVT::f32, &ARM::SPRRegClass);
536 addRegisterClass(MVT::f64, &ARM::DPRRegClass);
537
538 if (!Subtarget->hasVFP2Base()) {
539 setAllExpand(MVT::f32);
540 } else {
543
546 setOperationAction(Op, MVT::f32, Legal);
547 }
548 if (!Subtarget->hasFP64()) {
549 setAllExpand(MVT::f64);
550 } else {
553 setOperationAction(Op, MVT::f64, Legal);
554
556 }
557 }
558
559 if (Subtarget->hasFullFP16()) {
562 setOperationAction(Op, MVT::f16, Legal);
563
564 addRegisterClass(MVT::f16, &ARM::HPRRegClass);
567
572 }
573
574 if (Subtarget->hasBF16()) {
575 addRegisterClass(MVT::bf16, &ARM::HPRRegClass);
576 setAllExpand(MVT::bf16);
577 if (!Subtarget->hasFullFP16()) {
581 }
586 } else {
591 }
592
594 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
595 setTruncStoreAction(VT, InnerVT, Expand);
596 addAllExtLoads(VT, InnerVT, Expand);
597 }
598
601
603 }
604
605 if (!Subtarget->isThumb1Only() && !Subtarget->hasV8_1MMainlineOps())
607
608 if (!Subtarget->hasV8_1MMainlineOps())
610
611 if (!Subtarget->isThumb1Only())
613
616
619
620 if (Subtarget->hasMVEIntegerOps())
621 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
622
623 // Combine low-overhead loop intrinsics so that we can lower i1 types.
624 if (Subtarget->hasLOB()) {
626 }
627
628 if (Subtarget->hasNEON()) {
629 addDRTypeForNEON(MVT::v2f32);
630 addDRTypeForNEON(MVT::v8i8);
631 addDRTypeForNEON(MVT::v4i16);
632 addDRTypeForNEON(MVT::v2i32);
633 addDRTypeForNEON(MVT::v1i64);
634
635 addQRTypeForNEON(MVT::v4f32);
636 addQRTypeForNEON(MVT::v2f64);
637 addQRTypeForNEON(MVT::v16i8);
638 addQRTypeForNEON(MVT::v8i16);
639 addQRTypeForNEON(MVT::v4i32);
640 addQRTypeForNEON(MVT::v2i64);
641
642 if (Subtarget->hasFullFP16()) {
643 addQRTypeForNEON(MVT::v8f16);
644 addDRTypeForNEON(MVT::v4f16);
645 }
646
647 if (Subtarget->hasBF16()) {
648 addQRTypeForNEON(MVT::v8bf16);
649 addDRTypeForNEON(MVT::v4bf16);
650 }
651 }
652
653 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
654 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
655 // none of Neon, MVE or VFP supports any arithmetic operations on it.
656 setOperationAction(ISD::FADD, MVT::v2f64, Expand);
657 setOperationAction(ISD::FSUB, MVT::v2f64, Expand);
658 setOperationAction(ISD::FMUL, MVT::v2f64, Expand);
659 // FIXME: Code duplication: FDIV and FREM are expanded always, see
660 // ARMTargetLowering::addTypeForNEON method for details.
661 setOperationAction(ISD::FDIV, MVT::v2f64, Expand);
662 setOperationAction(ISD::FREM, MVT::v2f64, Expand);
663 // FIXME: Create unittest.
664 // In another words, find a way when "copysign" appears in DAG with vector
665 // operands.
667 // FIXME: Code duplication: SETCC has custom operation action, see
668 // ARMTargetLowering::addTypeForNEON method for details.
670 // FIXME: Create unittest for FNEG and for FABS.
671 setOperationAction(ISD::FNEG, MVT::v2f64, Expand);
672 setOperationAction(ISD::FABS, MVT::v2f64, Expand);
674 setOperationAction(ISD::FSIN, MVT::v2f64, Expand);
675 setOperationAction(ISD::FCOS, MVT::v2f64, Expand);
676 setOperationAction(ISD::FTAN, MVT::v2f64, Expand);
677 setOperationAction(ISD::FPOW, MVT::v2f64, Expand);
678 setOperationAction(ISD::FLOG, MVT::v2f64, Expand);
681 setOperationAction(ISD::FEXP, MVT::v2f64, Expand);
690 setOperationAction(ISD::FMA, MVT::v2f64, Expand);
691 }
692
693 if (Subtarget->hasNEON()) {
694 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
695 // supported for v4f32.
697 setOperationAction(ISD::FSIN, MVT::v4f32, Expand);
698 setOperationAction(ISD::FCOS, MVT::v4f32, Expand);
699 setOperationAction(ISD::FTAN, MVT::v4f32, Expand);
700 setOperationAction(ISD::FPOW, MVT::v4f32, Expand);
701 setOperationAction(ISD::FLOG, MVT::v4f32, Expand);
704 setOperationAction(ISD::FEXP, MVT::v4f32, Expand);
713
714 // Mark v2f32 intrinsics.
716 setOperationAction(ISD::FSIN, MVT::v2f32, Expand);
717 setOperationAction(ISD::FCOS, MVT::v2f32, Expand);
718 setOperationAction(ISD::FTAN, MVT::v2f32, Expand);
719 setOperationAction(ISD::FPOW, MVT::v2f32, Expand);
720 setOperationAction(ISD::FLOG, MVT::v2f32, Expand);
723 setOperationAction(ISD::FEXP, MVT::v2f32, Expand);
732
735 setOperationAction(Op, MVT::v4f16, Expand);
736 setOperationAction(Op, MVT::v8f16, Expand);
737 }
738
739 // Neon does not support some operations on v1i64 and v2i64 types.
740 setOperationAction(ISD::MUL, MVT::v1i64, Expand);
741 // Custom handling for some quad-vector types to detect VMULL.
742 setOperationAction(ISD::MUL, MVT::v8i16, Custom);
743 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
744 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
745 // Custom handling for some vector types to avoid expensive expansions
746 setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
748 setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
750 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
751 // a destination type that is wider than the source, and nor does
752 // it have a FP_TO_[SU]INT instruction with a narrower destination than
753 // source.
762
765
766 // NEON does not have single instruction CTPOP for vectors with element
767 // types wider than 8-bits. However, custom lowering can leverage the
768 // v8i8/v16i8 vcnt instruction.
775
776 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
777 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
778
779 // NEON does not have single instruction CTTZ for vectors.
781 setOperationAction(ISD::CTTZ, MVT::v4i16, Custom);
782 setOperationAction(ISD::CTTZ, MVT::v2i32, Custom);
783 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
784
785 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
786 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
787 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
788 setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);
789
794
799
803 }
804
805 // NEON only has FMA instructions as of VFP4.
806 if (!Subtarget->hasVFP4Base()) {
807 setOperationAction(ISD::FMA, MVT::v2f32, Expand);
808 setOperationAction(ISD::FMA, MVT::v4f32, Expand);
809 }
810
813
814 // It is legal to extload from v4i8 to v4i16 or v4i32.
815 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
816 MVT::v2i32}) {
821 }
822 }
823
824 for (auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
825 MVT::v4i32}) {
830 }
831 }
832
833 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
840 }
841 if (Subtarget->hasMVEIntegerOps()) {
844 ISD::SETCC});
845 }
846 if (Subtarget->hasMVEFloatOps()) {
848 }
849
850 if (!Subtarget->hasFP64()) {
851 // When targeting a floating-point unit with only single-precision
852 // operations, f64 is legal for the few double-precision instructions which
853 // are present However, no double-precision operations other than moves,
854 // loads and stores are provided by the hardware.
891 }
892
893 // STRICT_(U/S)INT_TO_FP specifically use the input MVT to register with
894 // setOperationAction() as opposed to other opcodes that use the output MVT
895 // All inputs should be i32 due to type legalization
898
901
902 if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) {
903 if (Subtarget->hasFullFP16()) {
906 }
907 }
908
909 if (Subtarget->hasFP16()) {
911 }
912
917
918 computeRegisterProperties(Subtarget->getRegisterInfo());
919
920 // ARM does not have floating-point extending loads.
921 for (MVT VT : MVT::fp_valuetypes()) {
922 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
923 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
924 setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
925 }
926
927 // ... or truncating stores
928 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
929 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
930 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
931 setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
932 setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
933
934 // ARM does not have i1 sign extending load.
935 for (MVT VT : MVT::integer_valuetypes())
937
938 // ARM supports all 4 flavors of integer indexed load / store.
939 if (!Subtarget->isThumb1Only()) {
940 for (unsigned im = (unsigned)ISD::PRE_INC;
942 setIndexedLoadAction(im, MVT::i1, Legal);
943 setIndexedLoadAction(im, MVT::i8, Legal);
944 setIndexedLoadAction(im, MVT::i16, Legal);
945 setIndexedLoadAction(im, MVT::i32, Legal);
946 setIndexedStoreAction(im, MVT::i1, Legal);
947 setIndexedStoreAction(im, MVT::i8, Legal);
948 setIndexedStoreAction(im, MVT::i16, Legal);
949 setIndexedStoreAction(im, MVT::i32, Legal);
950 }
951 } else {
952 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}.
955 }
956
957 // Custom loads/stores to possible use __aeabi_uread/write*
958 if (TT.isTargetAEABI() && !Subtarget->allowsUnalignedMem()) {
963 }
964
969
970 if (!Subtarget->isThumb1Only()) {
973 }
974
979 if (Subtarget->hasDSP()) {
988 }
989 if (Subtarget->hasBaseDSP()) {
992 }
993
994 // i64 operation support.
997 if (Subtarget->isThumb1Only()) {
1000 }
1001 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
1002 || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
1004
1014
1015 // MVE lowers 64 bit shifts to lsll and lsrl
1016 // assuming that ISD::SRL and SRA of i64 are already marked custom
1017 if (Subtarget->hasMVEIntegerOps())
1019
1020 // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1.
1021 if (Subtarget->isThumb1Only()) {
1025 }
1026
1027 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
1029
1030 // ARM does not have ROTL.
1035 }
1037 // TODO: These two should be set to LibCall, but this currently breaks
1038 // the Linux kernel build. See #101786.
1041 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
1044 }
1045
1046 // @llvm.readcyclecounter requires the Performance Monitors extension.
1047 // Default to the 0 expansion on unsupported platforms.
1048 // FIXME: Technically there are older ARM CPUs that have
1049 // implementation-specific ways of obtaining this information.
1050 if (Subtarget->hasPerfMon())
1052
1053 // Only ARMv6 has BSWAP.
1054 if (!Subtarget->hasV6Ops())
1056
1057 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1058 : Subtarget->hasDivideInARMMode();
1059 if (!hasDivide) {
1060 // These are expanded into libcalls if the cpu doesn't have HW divider.
1063 }
1064
1065 if (TT.isOSWindows() && !Subtarget->hasDivideInThumbMode()) {
1068
1071 }
1072
1075
1076 // Register based DivRem for AEABI (RTABI 4.2)
1077 if (TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
1078 TT.isTargetMuslAEABI() || TT.isOSFuchsia() || TT.isOSWindows()) {
1081 HasStandaloneRem = false;
1082
1087 } else {
1090 }
1091
1096
1097 setOperationAction(ISD::TRAP, MVT::Other, Legal);
1099
1100 // Use the default implementation.
1102 setOperationAction(ISD::VAARG, MVT::Other, Expand);
1104 setOperationAction(ISD::VAEND, MVT::Other, Expand);
1107
1108 if (TT.isOSWindows())
1110 else
1112
1113 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
1114 // the default expansion.
1115 InsertFencesForAtomic = false;
1116 if (hasNativeAtomics(*Subtarget)) {
1117 // ATOMIC_FENCE needs custom lowering; the others should have been expanded
1118 // to ldrex/strex loops already.
1120 if (!Subtarget->isThumb() || !Subtarget->isMClass())
1122
1123 // On v8, we have particularly efficient implementations of atomic fences
1124 // if they can be combined with nearby atomic loads and stores.
1125 if (!Subtarget->hasAcquireRelease() ||
1126 getTargetMachine().getOptLevel() == CodeGenOptLevel::None) {
1127 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
1128 InsertFencesForAtomic = true;
1129 }
1130 } else {
1131 // If there's anything we can use as a barrier, go through custom lowering
1132 // for ATOMIC_FENCE.
1133 // If target has DMB in thumb, Fences can be inserted.
1134 if (Subtarget->hasDataBarrier())
1135 InsertFencesForAtomic = true;
1136
1138 Subtarget->hasAnyDataBarrier() ? Custom : Expand);
1139
1140 // Set them all for libcall, which will force libcalls.
1146 {MVT::i8, MVT::i16, MVT::i32, MVT::i64}, LibCall);
1148 LibCall);
1149 }
1150
1151 // Compute supported atomic widths.
1152 if (TT.isOSLinux() || (!Subtarget->isMClass() && Subtarget->hasV6Ops())) {
1153 // For targets where __sync_* routines are reliably available, we use them
1154 // if necessary.
1155 //
1156 // ARM Linux always supports 64-bit atomics through kernel-assisted atomic
1157 // routines (kernel 3.1 or later). FIXME: Not with compiler-rt?
1158 //
1159 // ARMv6 targets have native instructions in ARM mode. For Thumb mode,
1160 // such targets should provide __sync_* routines, which use the ARM mode
1161 // instructions. (ARMv6 doesn't have dmb, but it has an equivalent
1162 // encoding; see ARMISD::MEMBARRIER_MCR.)
1164 } else if ((Subtarget->isMClass() && Subtarget->hasV8MBaselineOps()) ||
1165 Subtarget->hasForced32BitAtomics()) {
1166 // Cortex-M (besides Cortex-M0) have 32-bit atomics.
1168 } else {
1169 // We can't assume anything about other targets; just use libatomic
1170 // routines.
1172 }
1173
1175
1177
1178 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
1179 if (!Subtarget->hasV6Ops()) {
1182 }
1184
1185 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1186 !Subtarget->isThumb1Only()) {
1187 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
1188 // iff target supports vfp2.
1198 }
1199
1200 // We want to custom lower some of our intrinsics.
1205
1215 if (Subtarget->hasFullFP16()) {
1219 }
1220
1222
1225 if (Subtarget->hasFullFP16())
1229 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
1230
1231 // We don't support sin/cos/fmod/copysign/pow
1240 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() &&
1241 !Subtarget->isThumb1Only()) {
1244 }
1247
1248 if (!Subtarget->hasVFP4Base()) {
1251 }
1252
1253 // Various VFP goodness
1254 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
1255 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
1256 if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) {
1261 }
1262
1263 // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
1264 if (!Subtarget->hasFP16()) {
1269 }
1270
1271 // Strict floating-point comparisons need custom lowering.
1278 }
1279
1280 // FP-ARMv8 implements a lot of rounding-like FP operations.
1281 if (Subtarget->hasFPARMv8Base()) {
1282 for (auto Op :
1289 setOperationAction(Op, MVT::f32, Legal);
1290
1291 if (Subtarget->hasFP64())
1292 setOperationAction(Op, MVT::f64, Legal);
1293 }
1294
1295 if (Subtarget->hasNEON()) {
1300 }
1301 }
1302
1303 // FP16 often need to be promoted to call lib functions
1304 // clang-format off
1305 if (Subtarget->hasFullFP16()) {
1309
1310 for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI,
1324 setOperationAction(Op, MVT::f16, Promote);
1325 }
1326
1327 // Round-to-integer need custom lowering for fp16, as Promote doesn't work
1328 // because the result type is integer.
1330 setOperationAction(Op, MVT::f16, Custom);
1331
1337 setOperationAction(Op, MVT::f16, Legal);
1338 }
1339 // clang-format on
1340 }
1341
1342 if (Subtarget->hasNEON()) {
1343 // vmin and vmax aren't available in a scalar form, so we can use
1344 // a NEON instruction with an undef lane instead.
1353
1354 if (Subtarget->hasV8Ops()) {
1355 for (auto Op :
1359 setOperationAction(Op, MVT::v2f32, Legal);
1360 setOperationAction(Op, MVT::v4f32, Legal);
1361 }
1362 }
1363
1364 if (Subtarget->hasFullFP16()) {
1369
1374
1375 for (auto Op :
1379 setOperationAction(Op, MVT::v4f16, Legal);
1380 setOperationAction(Op, MVT::v8f16, Legal);
1381 }
1382 }
1383 }
1384
1385 // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined. MinGW has
1386 // it, but it's just a wrapper around ldexp.
1387 if (TT.isOSWindows()) {
1389 if (isOperationExpand(Op, MVT::f32))
1390 setOperationAction(Op, MVT::f32, Promote);
1391 }
1392
1393 // LegalizeDAG currently can't expand fp16 LDEXP/FREXP on targets where i16
1394 // isn't legal.
1396 if (isOperationExpand(Op, MVT::f16))
1397 setOperationAction(Op, MVT::f16, Promote);
1398
1399 // We have target-specific dag combine patterns for the following nodes:
1400 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine
1403
1404 if (Subtarget->hasMVEIntegerOps())
1406
1407 if (Subtarget->hasV6Ops())
1409 if (Subtarget->isThumb1Only())
1411 // Attempt to lower smin/smax to ssat/usat
1412 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1413 Subtarget->isThumb2()) {
1415 }
1416
1418
1419 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
1420 !Subtarget->hasVFP2Base() || Subtarget->hasMinSize())
1422 else
1424
1425 //// temporary - rewrite interface to use type
1428 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
1430 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
1432
1433 // On ARM arguments smaller than 4 bytes are extended, so all arguments
1434 // are at least 4 bytes aligned.
1436
1437 // Prefer likely predicted branches to selects on out-of-order cores.
1438 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
1439
1441 Align::fromLog2(Subtarget->getPreferBranchLogAlignment()));
1443 Align::fromLog2(Subtarget->getPreferBranchLogAlignment()));
1444
1445 setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
1446
1447 IsStrictFPEnabled = true;
1448}
1449
1451 return Subtarget->useSoftFloat();
1452}
1453
1455 return !Subtarget->isThumb1Only() && VT.getSizeInBits() <= 32;
1456}
1457
1458// FIXME: It might make sense to define the representative register class as the
1459// nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1460// a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1461// SPR's representative would be DPR_VFP2. This should work well if register
1462// pressure tracking were modified such that a register use would increment the
1463// pressure of the register class's representative and all of it's super
1464// classes' representatives transitively. We have not implemented this because
1465// of the difficulty prior to coalescing of modeling operand register classes
1466// due to the common occurrence of cross class copies and subregister insertions
1467// and extractions.
1468std::pair<const TargetRegisterClass *, uint8_t>
1470 MVT VT) const {
1471 const TargetRegisterClass *RRC = nullptr;
1472 uint8_t Cost = 1;
1473 switch (VT.SimpleTy) {
1474 default:
1476 // Use DPR as representative register class for all floating point
1477 // and vector types. Since there are 32 SPR registers and 32 DPR registers so
1478 // the cost is 1 for both f32 and f64.
1479 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
1480 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
1481 RRC = &ARM::DPRRegClass;
1482 // When NEON is used for SP, only half of the register file is available
1483 // because operations that define both SP and DP results will be constrained
1484 // to the VFP2 class (D0-D15). We currently model this constraint prior to
1485 // coalescing by double-counting the SP regs. See the FIXME above.
1486 if (Subtarget->useNEONForSinglePrecisionFP())
1487 Cost = 2;
1488 break;
1489 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1490 case MVT::v4f32: case MVT::v2f64:
1491 RRC = &ARM::DPRRegClass;
1492 Cost = 2;
1493 break;
1494 case MVT::v4i64:
1495 RRC = &ARM::DPRRegClass;
1496 Cost = 4;
1497 break;
1498 case MVT::v8i64:
1499 RRC = &ARM::DPRRegClass;
1500 Cost = 8;
1501 break;
1502 }
1503 return std::make_pair(RRC, Cost);
1504}
1505
1507 EVT VT) const {
1508 if (!VT.isVector())
1509 return getPointerTy(DL);
1510
1511 // MVE has a predicate register.
1512 if (Subtarget->hasMVEIntegerOps())
1513 return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount());
1514
1516}
1517
1518/// getRegClassFor - Return the register class that should be used for the
1519/// specified value type.
1520const TargetRegisterClass *
1521ARMTargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
1522 (void)isDivergent;
1523 // Map v4i64 to QQ registers but do not make the type legal. Similarly map
1524 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to
1525 // load / store 4 to 8 consecutive NEON D registers, or 2 to 4 consecutive
1526 // MVE Q registers.
1527 if (Subtarget->hasNEON()) {
1528 if (VT == MVT::v4i64)
1529 return &ARM::QQPRRegClass;
1530 if (VT == MVT::v8i64)
1531 return &ARM::QQQQPRRegClass;
1532 }
1533 if (Subtarget->hasMVEIntegerOps()) {
1534 if (VT == MVT::v4i64)
1535 return &ARM::MQQPRRegClass;
1536 if (VT == MVT::v8i64)
1537 return &ARM::MQQQQPRRegClass;
1538 }
1540}
1541
1542// memcpy, and other memory intrinsics, typically tries to use LDM/STM if the
1543// source/dest is aligned and the copy size is large enough. We therefore want
1544// to align such objects passed to memory intrinsics.
1546 Align &PrefAlign) const {
1547 if (!isa<MemIntrinsic>(CI))
1548 return false;
1549 MinSize = 8;
1550 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1
1551 // cycle faster than 4-byte aligned LDM.
1552 PrefAlign =
1553 (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? Align(8) : Align(4));
1554 return true;
1555}
1556
1557// Create a fast isel object.
1559 FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo,
1560 const LibcallLoweringInfo *libcallLowering) const {
1561 return ARM::createFastISel(funcInfo, libInfo, libcallLowering);
1562}
1563
1565 unsigned NumVals = N->getNumValues();
1566 if (!NumVals)
1567 return Sched::RegPressure;
1568
1569 for (unsigned i = 0; i != NumVals; ++i) {
1570 EVT VT = N->getValueType(i);
1571 if (VT == MVT::Glue || VT == MVT::Other)
1572 continue;
1573 if (VT.isFloatingPoint() || VT.isVector())
1574 return Sched::ILP;
1575 }
1576
1577 if (!N->isMachineOpcode())
1578 return Sched::RegPressure;
1579
1580 // Load are scheduled for latency even if there instruction itinerary
1581 // is not available.
1582 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1583 const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
1584
1585 if (MCID.getNumDefs() == 0)
1586 return Sched::RegPressure;
1587 if (!Itins->isEmpty() &&
1588 Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2U)
1589 return Sched::ILP;
1590
1591 return Sched::RegPressure;
1592}
1593
1594//===----------------------------------------------------------------------===//
1595// Lowering Code
1596//===----------------------------------------------------------------------===//
1597
1598static bool isSRL16(const SDValue &Op) {
1599 if (Op.getOpcode() != ISD::SRL)
1600 return false;
1601 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
1602 return Const->getZExtValue() == 16;
1603 return false;
1604}
1605
1606static bool isSRA16(const SDValue &Op) {
1607 if (Op.getOpcode() != ISD::SRA)
1608 return false;
1609 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
1610 return Const->getZExtValue() == 16;
1611 return false;
1612}
1613
1614static bool isSHL16(const SDValue &Op) {
1615 if (Op.getOpcode() != ISD::SHL)
1616 return false;
1617 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
1618 return Const->getZExtValue() == 16;
1619 return false;
1620}
1621
1622// Check for a signed 16-bit value. We special case SRA because it makes it
1623// more simple when also looking for SRAs that aren't sign extending a
1624// smaller value. Without the check, we'd need to take extra care with
1625// checking order for some operations.
1626static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
1627 if (isSRA16(Op))
1628 return isSHL16(Op.getOperand(0));
1629 return DAG.ComputeNumSignBits(Op) == 17;
1630}
1631
1632/// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
1634 switch (CC) {
1635 default: llvm_unreachable("Unknown condition code!");
1636 case ISD::SETNE: return ARMCC::NE;
1637 case ISD::SETEQ: return ARMCC::EQ;
1638 case ISD::SETGT: return ARMCC::GT;
1639 case ISD::SETGE: return ARMCC::GE;
1640 case ISD::SETLT: return ARMCC::LT;
1641 case ISD::SETLE: return ARMCC::LE;
1642 case ISD::SETUGT: return ARMCC::HI;
1643 case ISD::SETUGE: return ARMCC::HS;
1644 case ISD::SETULT: return ARMCC::LO;
1645 case ISD::SETULE: return ARMCC::LS;
1646 }
1647}
1648
1649/// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
1651 ARMCC::CondCodes &CondCode2) {
1652 CondCode2 = ARMCC::AL;
1653 switch (CC) {
1654 default: llvm_unreachable("Unknown FP condition!");
1655 case ISD::SETEQ:
1656 case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
1657 case ISD::SETGT:
1658 case ISD::SETOGT: CondCode = ARMCC::GT; break;
1659 case ISD::SETGE:
1660 case ISD::SETOGE: CondCode = ARMCC::GE; break;
1661 case ISD::SETOLT: CondCode = ARMCC::MI; break;
1662 case ISD::SETOLE: CondCode = ARMCC::LS; break;
1663 case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
1664 case ISD::SETO: CondCode = ARMCC::VC; break;
1665 case ISD::SETUO: CondCode = ARMCC::VS; break;
1666 case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
1667 case ISD::SETUGT: CondCode = ARMCC::HI; break;
1668 case ISD::SETUGE: CondCode = ARMCC::PL; break;
1669 case ISD::SETLT:
1670 case ISD::SETULT: CondCode = ARMCC::LT; break;
1671 case ISD::SETLE:
1672 case ISD::SETULE: CondCode = ARMCC::LE; break;
1673 case ISD::SETNE:
1674 case ISD::SETUNE: CondCode = ARMCC::NE; break;
1675 }
1676}
1677
1678//===----------------------------------------------------------------------===//
1679// Calling Convention Implementation
1680//===----------------------------------------------------------------------===//
1681
1682/// getEffectiveCallingConv - Get the effective calling convention, taking into
1683/// account presence of floating point hardware and calling convention
1684/// limitations, such as support for variadic functions.
1687 bool isVarArg) const {
1688 switch (CC) {
1689 default:
1690 // Unknown CCs are rejected when calling convention lowering is required.
1693 case CallingConv::GHC:
1695 return CC;
1701 case CallingConv::Swift:
1704 case CallingConv::C:
1705 case CallingConv::Tail:
1706 if (!Subtarget->isAAPCS_ABI())
1707 return CallingConv::ARM_APCS;
1708 else if (Subtarget->isTargetHardFloat() && !isVarArg)
1710 else
1712 case CallingConv::Fast:
1714 if (!Subtarget->isAAPCS_ABI()) {
1715 if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() && !isVarArg)
1716 return CallingConv::Fast;
1717 return CallingConv::ARM_APCS;
1718 } else if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() &&
1719 !Subtarget->useSoftFloat() && !isVarArg)
1721 else
1723 }
1724}
1725
1727 bool isVarArg) const {
1728 return CCAssignFnForNode(CC, false, isVarArg);
1729}
1730
1732 bool isVarArg) const {
1733 return CCAssignFnForNode(CC, true, isVarArg);
1734}
1735
1736/// CCAssignFnForNode - Selects the correct CCAssignFn for the given
1737/// CallingConvention.
1738CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
1739 bool Return,
1740 bool isVarArg) const {
1741 switch (getEffectiveCallingConv(CC, isVarArg)) {
1742 default:
1743 report_fatal_error("Unsupported calling convention");
1745 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS);
1747 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1749 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
1750 case CallingConv::Fast:
1751 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
1752 case CallingConv::GHC:
1753 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
1755 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1757 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1759 return (Return ? RetCC_ARM_AAPCS : CC_ARM_Win32_CFGuard_Check);
1760 }
1761}
1762
1763SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
1764 MVT LocVT, MVT ValVT, SDValue Val) const {
1765 Val = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocVT.getSizeInBits()),
1766 Val);
1767 if (Subtarget->hasFullFP16()) {
1768 Val = DAG.getNode(ARMISD::VMOVhr, dl, ValVT, Val);
1769 } else {
1770 Val = DAG.getNode(ISD::TRUNCATE, dl,
1771 MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
1772 Val = DAG.getNode(ISD::BITCAST, dl, ValVT, Val);
1773 }
1774 return Val;
1775}
1776
1777SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
1778 MVT LocVT, MVT ValVT,
1779 SDValue Val) const {
1780 if (Subtarget->hasFullFP16()) {
1781 Val = DAG.getNode(ARMISD::VMOVrh, dl,
1782 MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
1783 } else {
1784 Val = DAG.getNode(ISD::BITCAST, dl,
1785 MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
1786 Val = DAG.getNode(ISD::ZERO_EXTEND, dl,
1787 MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
1788 }
1789 return DAG.getNode(ISD::BITCAST, dl, LocVT, Val);
1790}
1791
1792/// LowerCallResult - Lower the result values of a call into the
1793/// appropriate copies out of appropriate physical registers.
1794SDValue ARMTargetLowering::LowerCallResult(
1795 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
1796 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
1797 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
1798 SDValue ThisVal, bool isCmseNSCall) const {
1799 // Assign locations to each value returned by this call.
1801 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
1802 *DAG.getContext());
1803 CCInfo.AnalyzeCallResult(Ins, CCAssignFnForReturn(CallConv, isVarArg));
1804
1805 // Copy all of the result registers out of their specified physreg.
1806 for (unsigned i = 0; i != RVLocs.size(); ++i) {
1807 CCValAssign VA = RVLocs[i];
1808
1809 // Pass 'this' value directly from the argument to return value, to avoid
1810 // reg unit interference
1811 if (i == 0 && isThisReturn) {
1812 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 &&
1813 "unexpected return calling convention register assignment");
1814 InVals.push_back(ThisVal);
1815 continue;
1816 }
1817
1818 SDValue Val;
1819 if (VA.needsCustom() &&
1820 (VA.getLocVT() == MVT::f64 || VA.getLocVT() == MVT::v2f64)) {
1821 // Handle f64 or half of a v2f64.
1822 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
1823 InGlue);
1824 Chain = Lo.getValue(1);
1825 InGlue = Lo.getValue(2);
1826 VA = RVLocs[++i]; // skip ahead to next loc
1827 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
1828 InGlue);
1829 Chain = Hi.getValue(1);
1830 InGlue = Hi.getValue(2);
1831 if (!Subtarget->isLittle())
1832 std::swap (Lo, Hi);
1833 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
1834
1835 if (VA.getLocVT() == MVT::v2f64) {
1836 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
1837 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
1838 DAG.getConstant(0, dl, MVT::i32));
1839
1840 VA = RVLocs[++i]; // skip ahead to next loc
1841 Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InGlue);
1842 Chain = Lo.getValue(1);
1843 InGlue = Lo.getValue(2);
1844 VA = RVLocs[++i]; // skip ahead to next loc
1845 Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InGlue);
1846 Chain = Hi.getValue(1);
1847 InGlue = Hi.getValue(2);
1848 if (!Subtarget->isLittle())
1849 std::swap (Lo, Hi);
1850 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
1851 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
1852 DAG.getConstant(1, dl, MVT::i32));
1853 }
1854 } else {
1855 Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(),
1856 InGlue);
1857 Chain = Val.getValue(1);
1858 InGlue = Val.getValue(2);
1859 }
1860
1861 switch (VA.getLocInfo()) {
1862 default: llvm_unreachable("Unknown loc info!");
1863 case CCValAssign::Full: break;
1864 case CCValAssign::BCvt:
1865 Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val);
1866 break;
1867 }
1868
1869 // f16 arguments have their size extended to 4 bytes and passed as if they
1870 // had been copied to the LSBs of a 32-bit register.
1871 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
1872 if (VA.needsCustom() &&
1873 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
1874 Val = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Val);
1875
1876 // On CMSE Non-secure Calls, call results (returned values) whose bitwidth
1877 // is less than 32 bits must be sign- or zero-extended after the call for
1878 // security reasons. Although the ABI mandates an extension done by the
1879 // callee, the latter cannot be trusted to follow the rules of the ABI.
1880 const ISD::InputArg &Arg = Ins[VA.getValNo()];
1881 if (isCmseNSCall && Arg.ArgVT.isScalarInteger() &&
1882 VA.getLocVT().isScalarInteger() && Arg.ArgVT.bitsLT(MVT::i32))
1883 Val = handleCMSEValue(Val, Arg, DAG, dl);
1884
1885 InVals.push_back(Val);
1886 }
1887
1888 return Chain;
1889}
1890
1891std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
1892 const SDLoc &dl, SelectionDAG &DAG, const CCValAssign &VA, SDValue StackPtr,
1893 bool IsTailCall, int SPDiff) const {
1894 SDValue DstAddr;
1895 MachinePointerInfo DstInfo;
1896 int32_t Offset = VA.getLocMemOffset();
1898
1899 if (IsTailCall) {
1900 Offset += SPDiff;
1901 auto PtrVT = getPointerTy(DAG.getDataLayout());
1902 int Size = VA.getLocVT().getFixedSizeInBits() / 8;
1903 int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
1904 DstAddr = DAG.getFrameIndex(FI, PtrVT);
1905 DstInfo =
1907 } else {
1908 SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
1909 DstAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
1910 StackPtr, PtrOff);
1911 DstInfo =
1913 }
1914
1915 return std::make_pair(DstAddr, DstInfo);
1916}
1917
1918// Returns the type of copying which is required to set up a byval argument to
1919// a tail-called function. This isn't needed for non-tail calls, because they
1920// always need the equivalent of CopyOnce, but tail-calls sometimes need two to
1921// avoid clobbering another argument (CopyViaTemp), and sometimes can be
1922// optimised to zero copies when forwarding an argument from the caller's
1923// caller (NoCopy).
1924ARMTargetLowering::ByValCopyKind ARMTargetLowering::ByValNeedsCopyForTailCall(
1925 SelectionDAG &DAG, SDValue Src, SDValue Dst, ISD::ArgFlagsTy Flags) const {
1926 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1927 ARMFunctionInfo *AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
1928
1929 // Globals are always safe to copy from.
1931 return CopyOnce;
1932
1933 // Can only analyse frame index nodes, conservatively assume we need a
1934 // temporary.
1935 auto *SrcFrameIdxNode = dyn_cast<FrameIndexSDNode>(Src);
1936 auto *DstFrameIdxNode = dyn_cast<FrameIndexSDNode>(Dst);
1937 if (!SrcFrameIdxNode || !DstFrameIdxNode)
1938 return CopyViaTemp;
1939
1940 int SrcFI = SrcFrameIdxNode->getIndex();
1941 int DstFI = DstFrameIdxNode->getIndex();
1942 assert(MFI.isFixedObjectIndex(DstFI) &&
1943 "byval passed in non-fixed stack slot");
1944
1945 int64_t SrcOffset = MFI.getObjectOffset(SrcFI);
1946 int64_t DstOffset = MFI.getObjectOffset(DstFI);
1947
1948 // If the source is in the local frame, then the copy to the argument memory
1949 // is always valid.
1950 bool FixedSrc = MFI.isFixedObjectIndex(SrcFI);
1951 if (!FixedSrc ||
1952 (FixedSrc && SrcOffset < -(int64_t)AFI->getArgRegsSaveSize()))
1953 return CopyOnce;
1954
1955 // In the case of byval arguments split between registers and the stack,
1956 // computeAddrForCallArg returns a FrameIndex which corresponds only to the
1957 // stack portion, but the Src SDValue will refer to the full value, including
1958 // the local stack memory that the register portion gets stored into. We only
1959 // need to compare them for equality, so normalise on the full value version.
1960 uint64_t RegSize = Flags.getByValSize() - MFI.getObjectSize(DstFI);
1961 DstOffset -= RegSize;
1962
1963 // If the value is already in the correct location, then no copying is
1964 // needed. If not, then we need to copy via a temporary.
1965 if (SrcOffset == DstOffset)
1966 return NoCopy;
1967 else
1968 return CopyViaTemp;
1969}
1970
1971void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
1972 SDValue Chain, SDValue &Arg,
1973 RegsToPassVector &RegsToPass,
1974 CCValAssign &VA, CCValAssign &NextVA,
1975 SDValue &StackPtr,
1976 SmallVectorImpl<SDValue> &MemOpChains,
1977 bool IsTailCall,
1978 int SPDiff) const {
1979 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
1980 DAG.getVTList(MVT::i32, MVT::i32), Arg);
1981 unsigned id = Subtarget->isLittle() ? 0 : 1;
1982 RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd.getValue(id)));
1983
1984 if (NextVA.isRegLoc())
1985 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id)));
1986 else {
1987 assert(NextVA.isMemLoc());
1988 if (!StackPtr.getNode())
1989 StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP,
1991
1992 SDValue DstAddr;
1993 MachinePointerInfo DstInfo;
1994 std::tie(DstAddr, DstInfo) =
1995 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
1996 MemOpChains.push_back(
1997 DAG.getStore(Chain, dl, fmrrd.getValue(1 - id), DstAddr, DstInfo));
1998 }
1999}
2000
2001static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
2002 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
2004}
2005
2006/// LowerCall - Lowering a call into a callseq_start <-
2007/// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
2008/// nodes.
2009SDValue
2010ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2011 SmallVectorImpl<SDValue> &InVals) const {
2012 SelectionDAG &DAG = CLI.DAG;
2013 SDLoc &dl = CLI.DL;
2014 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2015 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2016 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2017 SDValue Chain = CLI.Chain;
2018 SDValue Callee = CLI.Callee;
2019 bool &isTailCall = CLI.IsTailCall;
2020 CallingConv::ID CallConv = CLI.CallConv;
2021 bool doesNotRet = CLI.DoesNotReturn;
2022 bool isVarArg = CLI.IsVarArg;
2023 const CallBase *CB = CLI.CB;
2024
2026 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2027 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2028 MachineFunction::CallSiteInfo CSInfo;
2029 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
2030 bool isThisReturn = false;
2031 bool isCmseNSCall = false;
2032 bool isSibCall = false;
2033 bool PreferIndirect = false;
2034 bool GuardWithBTI = false;
2035
2036 // Analyze operands of the call, assigning locations to each operand.
2038 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2039 *DAG.getContext());
2040 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CallConv, isVarArg));
2041
2042 // Lower 'returns_twice' calls to a pseudo-instruction.
2043 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Attribute::ReturnsTwice) &&
2044 !Subtarget->noBTIAtReturnTwice())
2045 GuardWithBTI = AFI->branchTargetEnforcement();
2046
2047 // Set type id for call site info.
2048 setTypeIdForCallsiteInfo(CB, MF, CSInfo);
2049
2050 // Determine whether this is a non-secure function call.
2051 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr("cmse_nonsecure_call"))
2052 isCmseNSCall = true;
2053
2054 // Disable tail calls if they're not supported.
2055 if (!Subtarget->supportsTailCall())
2056 isTailCall = false;
2057
2058 // For both the non-secure calls and the returns from a CMSE entry function,
2059 // the function needs to do some extra work after the call, or before the
2060 // return, respectively, thus it cannot end with a tail call
2061 if (isCmseNSCall || AFI->isCmseNSEntryFunction())
2062 isTailCall = false;
2063
2064 if (isa<GlobalAddressSDNode>(Callee)) {
2065 // If we're optimizing for minimum size and the function is called three or
2066 // more times in this block, we can improve codesize by calling indirectly
2067 // as BLXr has a 16-bit encoding.
2068 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2069 if (CLI.CB) {
2070 auto *BB = CLI.CB->getParent();
2071 PreferIndirect = Subtarget->isThumb() && Subtarget->hasMinSize() &&
2072 count_if(GV->users(), [&BB](const User *U) {
2073 return isa<Instruction>(U) &&
2074 cast<Instruction>(U)->getParent() == BB;
2075 }) > 2;
2076 }
2077 }
2078 if (isTailCall) {
2079 // Check if it's really possible to do a tail call.
2080 isTailCall =
2081 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, PreferIndirect);
2082
2083 if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt &&
2084 CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail)
2085 isSibCall = true;
2086
2087 // We don't support GuaranteedTailCallOpt for ARM, only automatically
2088 // detected sibcalls.
2089 if (isTailCall)
2090 ++NumTailCalls;
2091 }
2092
2093 if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
2094 report_fatal_error("failed to perform tail call elimination on a call "
2095 "site marked musttail");
2096
2097 // Get a count of how many bytes are to be pushed on the stack.
2098 unsigned NumBytes = CCInfo.getStackSize();
2099
2100 // SPDiff is the byte offset of the call's argument area from the callee's.
2101 // Stores to callee stack arguments will be placed in FixedStackSlots offset
2102 // by this amount for a tail call. In a sibling call it must be 0 because the
2103 // caller will deallocate the entire stack and the callee still expects its
2104 // arguments to begin at SP+0. Completely unused for non-tail calls.
2105 int SPDiff = 0;
2106
2107 if (isTailCall && !isSibCall) {
2108 auto FuncInfo = MF.getInfo<ARMFunctionInfo>();
2109 unsigned NumReusableBytes = FuncInfo->getArgumentStackSize();
2110
2111 // Since callee will pop argument stack as a tail call, we must keep the
2112 // popped size 16-byte aligned.
2113 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
2114 assert(StackAlign && "data layout string is missing stack alignment");
2115 NumBytes = alignTo(NumBytes, *StackAlign);
2116
2117 // SPDiff will be negative if this tail call requires more space than we
2118 // would automatically have in our incoming argument space. Positive if we
2119 // can actually shrink the stack.
2120 SPDiff = NumReusableBytes - NumBytes;
2121
2122 // If this call requires more stack than we have available from
2123 // LowerFormalArguments, tell FrameLowering to reserve space for it.
2124 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (unsigned)-SPDiff)
2125 AFI->setArgRegsSaveSize(-SPDiff);
2126 }
2127
2128 if (isSibCall) {
2129 // For sibling tail calls, memory operands are available in our caller's stack.
2130 NumBytes = 0;
2131 } else {
2132 // Adjust the stack pointer for the new arguments...
2133 // These operations are automatically eliminated by the prolog/epilog pass
2134 Chain = DAG.getCALLSEQ_START(Chain, isTailCall ? 0 : NumBytes, 0, dl);
2135 }
2136
2138 DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout()));
2139
2140 RegsToPassVector RegsToPass;
2141 SmallVector<SDValue, 8> MemOpChains;
2142
2143 // A musttail call in a variadic function must receive the caller's unnamed
2144 // register arguments unchanged. The forwarded set is disjoint from the named
2145 // arguments handled below.
2146 if (isVarArg && CLI.CB && CLI.CB->isMustTailCall()) {
2147 const auto &Forwards = AFI->getForwardedMustTailRegParms();
2148 for (const ForwardedRegister &F : Forwards) {
2149 SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
2150 RegsToPass.push_back(std::make_pair(F.PReg, Val));
2151 }
2152 }
2153
2154 // If we are doing a tail-call, any byval arguments will be written to stack
2155 // space which was used for incoming arguments. If any the values being used
2156 // are incoming byval arguments to this function, then they might be
2157 // overwritten by the stores of the outgoing arguments. To avoid this, we
2158 // need to make a temporary copy of them in local stack space, then copy back
2159 // to the argument area.
2160 DenseMap<unsigned, SDValue> ByValTemporaries;
2161 SDValue ByValTempChain;
2162 if (isTailCall) {
2163 SmallVector<SDValue, 8> ByValCopyChains;
2164 for (const CCValAssign &VA : ArgLocs) {
2165 unsigned ArgIdx = VA.getValNo();
2166 SDValue Src = OutVals[ArgIdx];
2167 ISD::ArgFlagsTy Flags = Outs[ArgIdx].Flags;
2168
2169 if (!Flags.isByVal())
2170 continue;
2171
2172 SDValue Dst;
2173 MachinePointerInfo DstInfo;
2174 std::tie(Dst, DstInfo) =
2175 computeAddrForCallArg(dl, DAG, VA, SDValue(), true, SPDiff);
2176 ByValCopyKind Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
2177
2178 if (Copy == NoCopy) {
2179 // If the argument is already at the correct offset on the stack
2180 // (because we are forwarding a byval argument from our caller), we
2181 // don't need any copying.
2182 continue;
2183 } else if (Copy == CopyOnce) {
2184 // If the argument is in our local stack frame, no other argument
2185 // preparation can clobber it, so we can copy it to the final location
2186 // later.
2187 ByValTemporaries[ArgIdx] = Src;
2188 } else {
2189 assert(Copy == CopyViaTemp && "unexpected enum value");
2190 // If we might be copying this argument from the outgoing argument
2191 // stack area, we need to copy via a temporary in the local stack
2192 // frame.
2193 int TempFrameIdx = MFI.CreateStackObject(
2194 Flags.getByValSize(), Flags.getNonZeroByValAlign(), false);
2195 SDValue Temp =
2196 DAG.getFrameIndex(TempFrameIdx, getPointerTy(DAG.getDataLayout()));
2197
2198 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
2199 SDValue AlignNode =
2200 DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
2201
2202 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
2203 SDValue Ops[] = {Chain, Temp, Src, SizeNode, AlignNode};
2204 ByValCopyChains.push_back(
2205 DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs, Ops));
2206 ByValTemporaries[ArgIdx] = Temp;
2207 }
2208 }
2209 if (!ByValCopyChains.empty())
2210 ByValTempChain =
2211 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, ByValCopyChains);
2212 }
2213
2214 // During a tail call, stores to the argument area must happen after all of
2215 // the function's incoming arguments have been loaded because they may alias.
2216 // This is done by folding in a TokenFactor from LowerFormalArguments, but
2217 // there's no point in doing so repeatedly so this tracks whether that's
2218 // happened yet.
2219 bool AfterFormalArgLoads = false;
2220
2221 // Walk the register/memloc assignments, inserting copies/loads. In the case
2222 // of tail call optimization, arguments are handled later.
2223 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2224 i != e;
2225 ++i, ++realArgIdx) {
2226 CCValAssign &VA = ArgLocs[i];
2227 SDValue Arg = OutVals[realArgIdx];
2228 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2229 bool isByVal = Flags.isByVal();
2230
2231 // Promote the value if needed.
2232 switch (VA.getLocInfo()) {
2233 default: llvm_unreachable("Unknown loc info!");
2234 case CCValAssign::Full: break;
2235 case CCValAssign::SExt:
2236 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
2237 break;
2238 case CCValAssign::ZExt:
2239 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
2240 break;
2241 case CCValAssign::AExt:
2242 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
2243 break;
2244 case CCValAssign::BCvt:
2245 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
2246 break;
2247 }
2248
2249 if (isTailCall && VA.isMemLoc() && !AfterFormalArgLoads) {
2250 Chain = DAG.getStackArgumentTokenFactor(Chain);
2251 if (ByValTempChain) {
2252 // In case of large byval copies, re-using the stackframe for tail-calls
2253 // can lead to overwriting incoming arguments on the stack. Force
2254 // loading these stack arguments before the copy to avoid that.
2255 SmallVector<SDValue, 8> IncomingLoad;
2256 for (unsigned I = 0; I < OutVals.size(); ++I) {
2257 if (Outs[I].Flags.isByVal())
2258 continue;
2259
2260 SDValue OutVal = OutVals[I];
2261 LoadSDNode *OutLN = dyn_cast_or_null<LoadSDNode>(OutVal);
2262 if (!OutLN)
2263 continue;
2264
2265 FrameIndexSDNode *FIN =
2267 if (!FIN)
2268 continue;
2269
2270 if (!MFI.isFixedObjectIndex(FIN->getIndex()))
2271 continue;
2272
2273 for (const CCValAssign &VA : ArgLocs) {
2274 if (VA.isMemLoc())
2275 IncomingLoad.push_back(OutVal.getValue(1));
2276 }
2277 }
2278
2279 // Update the chain to force loads for potentially clobbered argument
2280 // loads to happen before the byval copy.
2281 if (!IncomingLoad.empty()) {
2282 IncomingLoad.push_back(Chain);
2283 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, IncomingLoad);
2284 }
2285
2286 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chain,
2287 ByValTempChain);
2288 }
2289 AfterFormalArgLoads = true;
2290 }
2291
2292 // f16 arguments have their size extended to 4 bytes and passed as if they
2293 // had been copied to the LSBs of a 32-bit register.
2294 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
2295 if (VA.needsCustom() &&
2296 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
2297 Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
2298 } else {
2299 // f16 arguments could have been extended prior to argument lowering.
2300 // Mask them arguments if this is a CMSE nonsecure call.
2301 auto ArgVT = Outs[realArgIdx].ArgVT;
2302 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2303 auto LocBits = VA.getLocVT().getSizeInBits();
2304 auto MaskValue = APInt::getLowBitsSet(LocBits, ArgVT.getSizeInBits());
2305 SDValue Mask =
2306 DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
2307 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
2308 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
2309 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
2310 }
2311 }
2312
2313 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
2314 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
2315 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2316 DAG.getConstant(0, dl, MVT::i32));
2317 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2318 DAG.getConstant(1, dl, MVT::i32));
2319
2320 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2321 StackPtr, MemOpChains, isTailCall, SPDiff);
2322
2323 VA = ArgLocs[++i]; // skip ahead to next loc
2324 if (VA.isRegLoc()) {
2325 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2326 StackPtr, MemOpChains, isTailCall, SPDiff);
2327 } else {
2328 assert(VA.isMemLoc());
2329 SDValue DstAddr;
2330 MachinePointerInfo DstInfo;
2331 std::tie(DstAddr, DstInfo) =
2332 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2333 MemOpChains.push_back(DAG.getStore(Chain, dl, Op1, DstAddr, DstInfo));
2334 }
2335 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
2336 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2337 StackPtr, MemOpChains, isTailCall, SPDiff);
2338 } else if (VA.isRegLoc()) {
2339 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
2340 Outs[0].VT == MVT::i32) {
2341 assert(VA.getLocVT() == MVT::i32 &&
2342 "unexpected calling convention register assignment");
2343 assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&
2344 "unexpected use of 'returned'");
2345 isThisReturn = true;
2346 }
2347 const TargetOptions &Options = DAG.getTarget().Options;
2348 if (Options.EmitCallSiteInfo)
2349 CSInfo.ArgRegPairs.emplace_back(VA.getLocReg(), i);
2350 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
2351 } else if (isByVal) {
2352 assert(VA.isMemLoc());
2353 unsigned offset = 0;
2354
2355 // True if this byval aggregate will be split between registers
2356 // and memory.
2357 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2358 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2359
2360 SDValue ByValSrc;
2361 bool NeedsStackCopy;
2362 if (auto It = ByValTemporaries.find(realArgIdx);
2363 It != ByValTemporaries.end()) {
2364 ByValSrc = It->second;
2365 NeedsStackCopy = true;
2366 } else {
2367 ByValSrc = Arg;
2368 NeedsStackCopy = !isTailCall;
2369 }
2370
2371 // If part of the argument is in registers, load them.
2372 if (CurByValIdx < ByValArgsCount) {
2373 unsigned RegBegin, RegEnd;
2374 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2375
2376 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2377 unsigned int i, j;
2378 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
2379 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
2380 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, ByValSrc, Const);
2381 SDValue Load =
2382 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo(),
2383 DAG.InferPtrAlign(AddArg));
2384 MemOpChains.push_back(Load.getValue(1));
2385 RegsToPass.push_back(std::make_pair(j, Load));
2386 }
2387
2388 // If parameter size outsides register area, "offset" value
2389 // helps us to calculate stack slot for remained part properly.
2390 offset = RegEnd - RegBegin;
2391
2392 CCInfo.nextInRegsParam();
2393 }
2394
2395 // If the memory part of the argument isn't already in the correct place
2396 // (which can happen with tail calls), copy it into the argument area.
2397 if (NeedsStackCopy && Flags.getByValSize() > 4 * offset) {
2398 auto PtrVT = getPointerTy(DAG.getDataLayout());
2399 SDValue Dst;
2400 MachinePointerInfo DstInfo;
2401 std::tie(Dst, DstInfo) =
2402 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2403 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
2404 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, ByValSrc, SrcOffset);
2405 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
2406 MVT::i32);
2407 SDValue AlignNode =
2408 DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
2409
2410 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
2411 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2412 MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
2413 Ops));
2414 }
2415 } else {
2416 assert(VA.isMemLoc());
2417 SDValue DstAddr;
2418 MachinePointerInfo DstInfo;
2419 std::tie(DstAddr, DstInfo) =
2420 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2421
2422 SDValue Store = DAG.getStore(Chain, dl, Arg, DstAddr, DstInfo);
2423 MemOpChains.push_back(Store);
2424 }
2425 }
2426
2427 if (!MemOpChains.empty())
2428 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
2429
2430 // Build a sequence of copy-to-reg nodes chained together with token chain
2431 // and flag operands which copy the outgoing args into the appropriate regs.
2432 SDValue InGlue;
2433 for (const auto &[Reg, N] : RegsToPass) {
2434 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
2435 InGlue = Chain.getValue(1);
2436 }
2437
2438 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
2439 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
2440 // node so that legalize doesn't hack it.
2441 bool isDirect = false;
2442
2443 const TargetMachine &TM = getTargetMachine();
2444 const Triple &TT = TM.getTargetTriple();
2445 const GlobalValue *GVal = nullptr;
2446 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
2447 GVal = G->getGlobal();
2448 bool isStub = !TM.shouldAssumeDSOLocal(GVal) && TT.isOSBinFormatMachO();
2449
2450 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
2451 bool isLocalARMFunc = false;
2452 auto PtrVt = getPointerTy(DAG.getDataLayout());
2453
2454 if (Subtarget->genLongCalls()) {
2455 bool isPIC = isPositionIndependent() && !TT.isOSWindows();
2456 if (isPIC && Subtarget->genExecuteOnly())
2457 reportFatalUsageError("long-calls with execute-only and "
2458 "position-independent code is not supported");
2459 if (Subtarget->isROPI())
2460 reportFatalUsageError("long-calls with ROPI is not currently supported");
2461
2462 // Handle a global address or an external symbol. If it's not one of
2463 // those, the target's already in a register, so we don't need to do
2464 // anything extra.
2465 if (isa<GlobalAddressSDNode>(Callee)) {
2466 if (Subtarget->genExecuteOnly()) {
2467 // Execute-only forbids constant pools in .text, so use movw/movt.
2468 // fPIC is not supported with execute-only.
2469 if (Subtarget->useMovt())
2470 ++NumMovwMovt;
2471 Callee = DAG.getNode(ARMISD::Wrapper, dl, PtrVt,
2472 DAG.getTargetGlobalAddress(GVal, dl, PtrVt));
2473 } else if (isPIC) {
2474 // PIC without execute-only: use GOT-based addressing.
2475 // DSO-local symbols use a plain PC-relative WrapperPIC;
2476 // non-DSO-local symbols additionally load the address from the GOT.
2478 GVal, dl, PtrVt, 0, GVal->isDSOLocal() ? 0 : ARMII::MO_GOT);
2479 Callee = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVt, G);
2480 if (!GVal->isDSOLocal())
2481 Callee =
2482 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), Callee,
2484 } else {
2485 // Neither execute-only nor PIC: load the address from a constant pool.
2486 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2487 ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(
2488 GVal, ARMPCLabelIndex, ARMCP::CPValue, 0);
2489
2490 // Get the address of the callee into a register
2491 SDValue Addr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2492 Addr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
2493 Callee = DAG.getLoad(
2494 PtrVt, dl, DAG.getEntryNode(), Addr,
2496 }
2497 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Callee)) {
2498 const char *Sym = S->getSymbol();
2499
2500 if (Subtarget->genExecuteOnly()) {
2501 // Execute-only forbids constant pools in .text, so use movw/movt.
2502 // fPIC is not supported with execute-only.
2503 if (Subtarget->useMovt())
2504 ++NumMovwMovt;
2505 Callee = DAG.getNode(ARMISD::Wrapper, dl, PtrVt,
2506 DAG.getTargetExternalSymbol(Sym, PtrVt, 0));
2507 } else if (isPIC) {
2508 // PIC without execute-only: load the symbol's address from the GOT via
2509 // a GOT_PREL constant pool entry consumed by a PICLDR.
2510 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
2511 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2512 ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
2513 *DAG.getContext(), Sym, ARMPCLabelIndex, PCAdj, ARMCP::GOT_PREL,
2514 /*AddCurrentAddress=*/true);
2515 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2516 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2517 SDValue GOTOffset = DAG.getLoad(
2518 PtrVt, dl, DAG.getEntryNode(), CPAddr,
2520 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2521 Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, GOTOffset, PICLabel);
2522 Callee =
2523 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), Callee,
2525 } else {
2526 // Neither execute-only nor PIC: load the address from a constant pool.
2527 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2528 ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
2529 *DAG.getContext(), Sym, ARMPCLabelIndex, 0);
2530
2531 // Get the address of the callee into a register
2532 SDValue Addr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2533 Addr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
2534 Callee = DAG.getLoad(
2535 PtrVt, dl, DAG.getEntryNode(), Addr,
2537 }
2538 }
2539 } else if (isa<GlobalAddressSDNode>(Callee)) {
2540 if (!PreferIndirect) {
2541 isDirect = true;
2542 bool isDef = GVal->isStrongDefinitionForLinker();
2543
2544 // ARM call to a local ARM function is predicable.
2545 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
2546 // tBX takes a register source operand.
2547 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2548 assert(TT.isOSBinFormatMachO() && "WrapperPIC use on non-MachO?");
2549 Callee = DAG.getNode(
2550 ARMISD::WrapperPIC, dl, PtrVt,
2551 DAG.getTargetGlobalAddress(GVal, dl, PtrVt, 0, ARMII::MO_NONLAZY));
2552 Callee = DAG.getLoad(
2553 PtrVt, dl, DAG.getEntryNode(), Callee,
2557 } else if (Subtarget->isTargetCOFF()) {
2558 assert(Subtarget->isTargetWindows() &&
2559 "Windows is the only supported COFF target");
2560 unsigned TargetFlags = ARMII::MO_NO_FLAG;
2561 if (GVal->hasDLLImportStorageClass())
2562 TargetFlags = ARMII::MO_DLLIMPORT;
2563 else if (!TM.shouldAssumeDSOLocal(GVal))
2564 TargetFlags = ARMII::MO_COFFSTUB;
2565 Callee = DAG.getTargetGlobalAddress(GVal, dl, PtrVt, /*offset=*/0,
2566 TargetFlags);
2567 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
2568 Callee =
2569 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(),
2570 DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee),
2572 } else {
2573 Callee = DAG.getTargetGlobalAddress(GVal, dl, PtrVt, 0, 0);
2574 }
2575 }
2576 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2577 isDirect = true;
2578 // tBX takes a register source operand.
2579 const char *Sym = S->getSymbol();
2580 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2581 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2582 ARMConstantPoolValue *CPV =
2584 ARMPCLabelIndex, 4);
2585 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2586 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2587 Callee = DAG.getLoad(
2588 PtrVt, dl, DAG.getEntryNode(), CPAddr,
2590 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2591 Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel);
2592 } else {
2593 Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, 0);
2594 }
2595 }
2596
2597 if (isCmseNSCall) {
2598 assert(!isARMFunc && !isDirect &&
2599 "Cannot handle call to ARM function or direct call");
2600 if (NumBytes > 0) {
2601 DAG.getContext()->diagnose(
2602 DiagnosticInfoUnsupported(DAG.getMachineFunction().getFunction(),
2603 "call to non-secure function would require "
2604 "passing arguments on stack",
2605 dl.getDebugLoc()));
2606 }
2607 if (isStructRet) {
2608 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
2610 "call to non-secure function would return value through pointer",
2611 dl.getDebugLoc()));
2612 }
2613 }
2614
2615 // FIXME: handle tail calls differently.
2616 unsigned CallOpc;
2617 if (Subtarget->isThumb()) {
2618 if (GuardWithBTI)
2619 CallOpc = ARMISD::t2CALL_BTI;
2620 else if (isCmseNSCall)
2621 CallOpc = ARMISD::tSECALL;
2622 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2623 CallOpc = ARMISD::CALL_NOLINK;
2624 else
2625 CallOpc = ARMISD::CALL;
2626 } else {
2627 if (!isDirect && !Subtarget->hasV5TOps())
2628 CallOpc = ARMISD::CALL_NOLINK;
2629 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2630 // Emit regular call when code size is the priority
2631 !Subtarget->hasMinSize())
2632 // "mov lr, pc; b _foo" to avoid confusing the RSP
2633 CallOpc = ARMISD::CALL_NOLINK;
2634 else
2635 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
2636 }
2637
2638 // We don't usually want to end the call-sequence here because we would tidy
2639 // the frame up *after* the call, however in the ABI-changing tail-call case
2640 // we've carefully laid out the parameters so that when sp is reset they'll be
2641 // in the correct location.
2642 if (isTailCall && !isSibCall) {
2643 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, dl);
2644 InGlue = Chain.getValue(1);
2645 }
2646
2647 std::vector<SDValue> Ops;
2648 Ops.push_back(Chain);
2649 Ops.push_back(Callee);
2650
2651 if (isTailCall) {
2652 Ops.push_back(DAG.getSignedTargetConstant(SPDiff, dl, MVT::i32));
2653 }
2654
2655 // Add argument registers to the end of the list so that they are known live
2656 // into the call.
2657 for (const auto &[Reg, N] : RegsToPass)
2658 Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
2659
2660 // Add a register mask operand representing the call-preserved registers.
2661 const uint32_t *Mask;
2662 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
2663 if (isThisReturn) {
2664 // For 'this' returns, use the R0-preserving mask if applicable
2665 Mask = ARI->getThisReturnPreservedMask(MF, CallConv);
2666 if (!Mask) {
2667 // Set isThisReturn to false if the calling convention is not one that
2668 // allows 'returned' to be modeled in this way, so LowerCallResult does
2669 // not try to pass 'this' straight through
2670 isThisReturn = false;
2671 Mask = ARI->getCallPreservedMask(MF, CallConv);
2672 }
2673 } else
2674 Mask = ARI->getCallPreservedMask(MF, CallConv);
2675
2676 assert(Mask && "Missing call preserved mask for calling convention");
2677 Ops.push_back(DAG.getRegisterMask(Mask));
2678
2679 if (InGlue.getNode())
2680 Ops.push_back(InGlue);
2681
2682 if (isTailCall) {
2684 SDValue Ret = DAG.getNode(ARMISD::TC_RETURN, dl, MVT::Other, Ops);
2685 if (CLI.CFIType)
2686 Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2687 DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
2688 DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
2689 return Ret;
2690 }
2691
2692 // Returns a chain and a flag for retval copy to use.
2693 Chain = DAG.getNode(CallOpc, dl, {MVT::Other, MVT::Glue}, Ops);
2694 if (CLI.CFIType)
2695 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2696 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
2697 InGlue = Chain.getValue(1);
2698 DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
2699
2700 // If we're guaranteeing tail-calls will be honoured, the callee must
2701 // pop its own argument stack on return. But this call is *not* a tail call so
2702 // we need to undo that after it returns to restore the status-quo.
2703 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
2704 uint64_t CalleePopBytes =
2705 canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : -1U;
2706
2707 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
2708 if (!Ins.empty())
2709 InGlue = Chain.getValue(1);
2710
2711 // Handle result values, copying them out of physregs into vregs that we
2712 // return.
2713 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2714 InVals, isThisReturn,
2715 isThisReturn ? OutVals[0] : SDValue(), isCmseNSCall);
2716}
2717
2718/// HandleByVal - Every parameter *after* a byval parameter is passed
2719/// on the stack. Remember the next parameter register to allocate,
2720/// and then confiscate the rest of the parameter registers to insure
2721/// this.
2722void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
2723 Align Alignment) const {
2724 // Byval (as with any stack) slots are always at least 4 byte aligned.
2725 Alignment = std::max(Alignment, Align(4));
2726
2727 MCRegister Reg = State->AllocateReg(GPRArgRegs);
2728 if (!Reg)
2729 return;
2730
2731 unsigned AlignInRegs = Alignment.value() / 4;
2732 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs;
2733 for (unsigned i = 0; i < Waste; ++i)
2734 Reg = State->AllocateReg(GPRArgRegs);
2735
2736 if (!Reg)
2737 return;
2738
2739 unsigned Excess = 4 * (ARM::R4 - Reg);
2740
2741 // Special case when NSAA != SP and parameter size greater than size of
2742 // all remained GPR regs. In that case we can't split parameter, we must
2743 // send it to stack. We also must set NCRN to R4, so waste all
2744 // remained registers.
2745 const unsigned NSAAOffset = State->getStackSize();
2746 if (NSAAOffset != 0 && Size > Excess) {
2747 while (State->AllocateReg(GPRArgRegs))
2748 ;
2749 return;
2750 }
2751
2752 // First register for byval parameter is the first register that wasn't
2753 // allocated before this method call, so it would be "reg".
2754 // If parameter is small enough to be saved in range [reg, r4), then
2755 // the end (first after last) register would be reg + param-size-in-regs,
2756 // else parameter would be splitted between registers and stack,
2757 // end register would be r4 in this case.
2758 unsigned ByValRegBegin = Reg;
2759 unsigned ByValRegEnd = std::min<unsigned>(Reg + Size / 4, ARM::R4);
2760 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd);
2761 // Note, first register is allocated in the beginning of function already,
2762 // allocate remained amount of registers we need.
2763 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2764 State->AllocateReg(GPRArgRegs);
2765 // A byval parameter that is split between registers and memory needs its
2766 // size truncated here.
2767 // In the case where the entire structure fits in registers, we set the
2768 // size in memory to zero.
2769 Size = std::max<int>(Size - Excess, 0);
2770}
2771
2772/// IsEligibleForTailCallOptimization - Check whether the call is eligible
2773/// for tail call optimization. Targets which want to do tail call
2774/// optimization should implement this function. Note that this function also
2775/// processes musttail calls, so when this function returns false on a valid
2776/// musttail call, a fatal backend error occurs.
2777bool ARMTargetLowering::IsEligibleForTailCallOptimization(
2779 SmallVectorImpl<CCValAssign> &ArgLocs, const bool isIndirect) const {
2780 CallingConv::ID CalleeCC = CLI.CallConv;
2781 SDValue Callee = CLI.Callee;
2782 bool isVarArg = CLI.IsVarArg;
2783 const SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2784 const SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2785 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2786 const SelectionDAG &DAG = CLI.DAG;
2788 const Function &CallerF = MF.getFunction();
2789 CallingConv::ID CallerCC = CallerF.getCallingConv();
2790
2791 assert(Subtarget->supportsTailCall());
2792
2793 // Indirect tail-calls require a register to hold the target address. That
2794 // register must be:
2795 // * Allocatable (i.e. r0-r7 if the target is Thumb1).
2796 // * Not callee-saved, so must be one of r0-r3 or r12.
2797 // * Not used to hold an argument to the tail-called function, which might be
2798 // in r0-r3.
2799 // * Not used to hold the return address authentication code, which is in r12
2800 // if enabled.
2801 // Sometimes, no register matches all of these conditions, so we can't do a
2802 // tail-call.
2803 if (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect) {
2804 SmallSet<MCPhysReg, 5> AddressRegisters = {ARM::R0, ARM::R1, ARM::R2,
2805 ARM::R3};
2806 if (!(Subtarget->isThumb1Only() ||
2807 MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(true)))
2808 AddressRegisters.insert(ARM::R12);
2809 for (const CCValAssign &AL : ArgLocs)
2810 if (AL.isRegLoc())
2811 AddressRegisters.erase(AL.getLocReg());
2812 if (AddressRegisters.empty()) {
2813 LLVM_DEBUG(dbgs() << "false (no reg to hold function pointer)\n");
2814 return false;
2815 }
2816 }
2817
2818 // Look for obvious safe cases to perform tail call optimization that do not
2819 // require ABI changes. This is what gcc calls sibcall.
2820
2821 // Exception-handling functions need a special set of instructions to indicate
2822 // a return to the hardware. Tail-calling another function would probably
2823 // break this.
2824 if (CallerF.hasFnAttribute("interrupt")) {
2825 LLVM_DEBUG(dbgs() << "false (interrupt attribute)\n");
2826 return false;
2827 }
2828
2829 if (canGuaranteeTCO(CalleeCC,
2830 getTargetMachine().Options.GuaranteedTailCallOpt)) {
2831 LLVM_DEBUG(dbgs() << (CalleeCC == CallerCC ? "true" : "false")
2832 << " (guaranteed tail-call CC)\n");
2833 return CalleeCC == CallerCC;
2834 }
2835
2836 // Also avoid sibcall optimization if either caller or callee uses struct
2837 // return semantics.
2838 bool isCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
2839 bool isCallerStructRet = MF.getFunction().hasStructRetAttr();
2840 if (isCalleeStructRet != isCallerStructRet) {
2841 LLVM_DEBUG(dbgs() << "false (struct-ret)\n");
2842 return false;
2843 }
2844
2845 // Externally-defined functions with weak linkage should not be
2846 // tail-called on ARM when the OS does not support dynamic
2847 // pre-emption of symbols, as the AAELF spec requires normal calls
2848 // to undefined weak functions to be replaced with a NOP or jump to the
2849 // next instruction. The behaviour of branch instructions in this
2850 // situation (as used for tail calls) is implementation-defined, so we
2851 // cannot rely on the linker replacing the tail call with a return.
2852 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2853 const GlobalValue *GV = G->getGlobal();
2854 const Triple &TT = GV->getParent()->getTargetTriple();
2855 if (GV->hasExternalWeakLinkage() &&
2856 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
2857 TT.isOSBinFormatMachO())) {
2858 LLVM_DEBUG(dbgs() << "false (external weak linkage)\n");
2859 return false;
2860 }
2861 }
2862
2863 // Check that the call results are passed in the same way.
2864 LLVMContext &C = *DAG.getContext();
2866 getEffectiveCallingConv(CalleeCC, isVarArg),
2867 getEffectiveCallingConv(CallerCC, CallerF.isVarArg()), MF, C, Ins,
2868 CCAssignFnForReturn(CalleeCC, isVarArg),
2869 CCAssignFnForReturn(CallerCC, CallerF.isVarArg()))) {
2870 LLVM_DEBUG(dbgs() << "false (incompatible results)\n");
2871 return false;
2872 }
2873 // The callee has to preserve all registers the caller needs to preserve.
2874 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
2875 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
2876 if (CalleeCC != CallerCC) {
2877 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2878 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) {
2879 LLVM_DEBUG(dbgs() << "false (not all registers preserved)\n");
2880 return false;
2881 }
2882 }
2883
2884 // If Caller's vararg argument has been split between registers and stack, do
2885 // not perform tail call, since part of the argument is in caller's local
2886 // frame.
2887 const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
2888 if (CLI.IsVarArg && AFI_Caller->getArgRegsSaveSize()) {
2889 LLVM_DEBUG(dbgs() << "false (arg reg save area)\n");
2890 return false;
2891 }
2892
2893 // If the callee takes no arguments then go on to check the results of the
2894 // call.
2895 const MachineRegisterInfo &MRI = MF.getRegInfo();
2896 if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals)) {
2897 LLVM_DEBUG(dbgs() << "false (parameters in CSRs do not match)\n");
2898 return false;
2899 }
2900
2901 // If the stack arguments for this call do not fit into our own save area then
2902 // the call cannot be made tail.
2903 if (CCInfo.getStackSize() > AFI_Caller->getArgumentStackSize())
2904 return false;
2905
2906 LLVM_DEBUG(dbgs() << "true\n");
2907 return true;
2908}
2909
2910bool
2911ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
2912 MachineFunction &MF, bool isVarArg,
2914 LLVMContext &Context, const Type *RetTy) const {
2916 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2917 return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
2918}
2919
2921 const SDLoc &DL, SelectionDAG &DAG) {
2922 const MachineFunction &MF = DAG.getMachineFunction();
2923 const Function &F = MF.getFunction();
2924
2925 StringRef IntKind = F.getFnAttribute("interrupt").getValueAsString();
2926
2927 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset
2928 // version of the "preferred return address". These offsets affect the return
2929 // instruction if this is a return from PL1 without hypervisor extensions.
2930 // IRQ/FIQ: +4 "subs pc, lr, #4"
2931 // SWI: 0 "subs pc, lr, #0"
2932 // ABORT: +4 "subs pc, lr, #4"
2933 // UNDEF: +4/+2 "subs pc, lr, #0"
2934 // UNDEF varies depending on where the exception came from ARM or Thumb
2935 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0.
2936
2937 int64_t LROffset;
2938 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" ||
2939 IntKind == "ABORT")
2940 LROffset = 4;
2941 else if (IntKind == "SWI" || IntKind == "UNDEF")
2942 LROffset = 0;
2943 else
2944 report_fatal_error("Unsupported interrupt attribute. If present, value "
2945 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
2946
2947 RetOps.insert(RetOps.begin() + 1,
2948 DAG.getConstant(LROffset, DL, MVT::i32, false));
2949
2950 return DAG.getNode(ARMISD::INTRET_GLUE, DL, MVT::Other, RetOps);
2951}
2952
2953SDValue
2954ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2955 bool isVarArg,
2957 const SmallVectorImpl<SDValue> &OutVals,
2958 const SDLoc &dl, SelectionDAG &DAG) const {
2959 // CCValAssign - represent the assignment of the return value to a location.
2961
2962 // CCState - Info about the registers and stack slots.
2963 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2964 *DAG.getContext());
2965
2966 // Analyze outgoing return values.
2967 CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
2968
2969 SDValue Glue;
2971 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2972 bool isLittleEndian = Subtarget->isLittle();
2973
2975 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2976 AFI->setReturnRegsCount(RVLocs.size());
2977
2978 // Report error if cmse entry function returns structure through first ptr arg.
2979 if (AFI->isCmseNSEntryFunction() && MF.getFunction().hasStructRetAttr()) {
2980 // Note: using an empty SDLoc(), as the first line of the function is a
2981 // better place to report than the last line.
2982 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
2984 "secure entry function would return value through pointer",
2985 SDLoc().getDebugLoc()));
2986 }
2987
2988 // Copy the result values into the output registers.
2989 for (unsigned i = 0, realRVLocIdx = 0;
2990 i != RVLocs.size();
2991 ++i, ++realRVLocIdx) {
2992 CCValAssign &VA = RVLocs[i];
2993 assert(VA.isRegLoc() && "Can only return in registers!");
2994
2995 SDValue Arg = OutVals[realRVLocIdx];
2996 bool ReturnF16 = false;
2997
2998 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) {
2999 // Half-precision return values can be returned like this:
3000 //
3001 // t11 f16 = fadd ...
3002 // t12: i16 = bitcast t11
3003 // t13: i32 = zero_extend t12
3004 // t14: f32 = bitcast t13 <~~~~~~~ Arg
3005 //
3006 // to avoid code generation for bitcasts, we simply set Arg to the node
3007 // that produces the f16 value, t11 in this case.
3008 //
3009 if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) {
3010 SDValue ZE = Arg.getOperand(0);
3011 if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) {
3012 SDValue BC = ZE.getOperand(0);
3013 if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) {
3014 Arg = BC.getOperand(0);
3015 ReturnF16 = true;
3016 }
3017 }
3018 }
3019 }
3020
3021 switch (VA.getLocInfo()) {
3022 default: llvm_unreachable("Unknown loc info!");
3023 case CCValAssign::Full: break;
3024 case CCValAssign::BCvt:
3025 if (!ReturnF16)
3026 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
3027 break;
3028 }
3029
3030 // Mask f16 arguments if this is a CMSE nonsecure entry.
3031 auto RetVT = Outs[realRVLocIdx].ArgVT;
3032 if (AFI->isCmseNSEntryFunction() && (RetVT == MVT::f16)) {
3033 if (VA.needsCustom() && VA.getValVT() == MVT::f16) {
3034 Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
3035 } else {
3036 auto LocBits = VA.getLocVT().getSizeInBits();
3037 auto MaskValue = APInt::getLowBitsSet(LocBits, RetVT.getSizeInBits());
3038 SDValue Mask =
3039 DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
3040 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
3041 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
3042 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
3043 }
3044 }
3045
3046 if (VA.needsCustom() &&
3047 (VA.getLocVT() == MVT::v2f64 || VA.getLocVT() == MVT::f64)) {
3048 if (VA.getLocVT() == MVT::v2f64) {
3049 // Extract the first half and return it in two registers.
3050 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
3051 DAG.getConstant(0, dl, MVT::i32));
3052 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
3053 DAG.getVTList(MVT::i32, MVT::i32), Half);
3054
3055 Chain =
3056 DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3057 HalfGPRs.getValue(isLittleEndian ? 0 : 1), Glue);
3058 Glue = Chain.getValue(1);
3059 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3060 VA = RVLocs[++i]; // skip ahead to next loc
3061 Chain =
3062 DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3063 HalfGPRs.getValue(isLittleEndian ? 1 : 0), Glue);
3064 Glue = Chain.getValue(1);
3065 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3066 VA = RVLocs[++i]; // skip ahead to next loc
3067
3068 // Extract the 2nd half and fall through to handle it as an f64 value.
3069 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
3070 DAG.getConstant(1, dl, MVT::i32));
3071 }
3072 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is
3073 // available.
3074 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
3075 DAG.getVTList(MVT::i32, MVT::i32), Arg);
3076 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3077 fmrrd.getValue(isLittleEndian ? 0 : 1), Glue);
3078 Glue = Chain.getValue(1);
3079 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3080 VA = RVLocs[++i]; // skip ahead to next loc
3081 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3082 fmrrd.getValue(isLittleEndian ? 1 : 0), Glue);
3083 } else
3084 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Glue);
3085
3086 // Guarantee that all emitted copies are
3087 // stuck together, avoiding something bad.
3088 Glue = Chain.getValue(1);
3089 RetOps.push_back(DAG.getRegister(
3090 VA.getLocReg(), ReturnF16 ? Arg.getValueType() : VA.getLocVT()));
3091 }
3092 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
3093 const MCPhysReg *I =
3094 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
3095 if (I) {
3096 for (; *I; ++I) {
3097 if (ARM::GPRRegClass.contains(*I))
3098 RetOps.push_back(DAG.getRegister(*I, MVT::i32));
3099 else if (ARM::DPRRegClass.contains(*I))
3101 else
3102 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3103 }
3104 }
3105
3106 // Update chain and glue.
3107 RetOps[0] = Chain;
3108 if (Glue.getNode())
3109 RetOps.push_back(Glue);
3110
3111 // CPUs which aren't M-class use a special sequence to return from
3112 // exceptions (roughly, any instruction setting pc and cpsr simultaneously,
3113 // though we use "subs pc, lr, #N").
3114 //
3115 // M-class CPUs actually use a normal return sequence with a special
3116 // (hardware-provided) value in LR, so the normal code path works.
3117 if (DAG.getMachineFunction().getFunction().hasFnAttribute("interrupt") &&
3118 !Subtarget->isMClass()) {
3119 if (Subtarget->isThumb1Only())
3120 report_fatal_error("interrupt attribute is not supported in Thumb1");
3121 return LowerInterruptReturn(RetOps, dl, DAG);
3122 }
3123
3124 unsigned RetNode =
3125 AFI->isCmseNSEntryFunction() ? ARMISD::SERET_GLUE : ARMISD::RET_GLUE;
3126 return DAG.getNode(RetNode, dl, MVT::Other, RetOps);
3127}
3128
3129bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
3130 if (N->getNumValues() != 1)
3131 return false;
3132 if (!N->hasNUsesOfValue(1, 0))
3133 return false;
3134
3135 SDValue TCChain = Chain;
3136 SDNode *Copy = *N->user_begin();
3137 if (Copy->getOpcode() == ISD::CopyToReg) {
3138 // If the copy has a glue operand, we conservatively assume it isn't safe to
3139 // perform a tail call.
3140 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3141 return false;
3142 TCChain = Copy->getOperand(0);
3143 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) {
3144 SDNode *VMov = Copy;
3145 // f64 returned in a pair of GPRs.
3146 SmallPtrSet<SDNode*, 2> Copies;
3147 for (SDNode *U : VMov->users()) {
3148 if (U->getOpcode() != ISD::CopyToReg)
3149 return false;
3150 Copies.insert(U);
3151 }
3152 if (Copies.size() > 2)
3153 return false;
3154
3155 for (SDNode *U : VMov->users()) {
3156 SDValue UseChain = U->getOperand(0);
3157 if (Copies.count(UseChain.getNode()))
3158 // Second CopyToReg
3159 Copy = U;
3160 else {
3161 // We are at the top of this chain.
3162 // If the copy has a glue operand, we conservatively assume it
3163 // isn't safe to perform a tail call.
3164 if (U->getOperand(U->getNumOperands() - 1).getValueType() == MVT::Glue)
3165 return false;
3166 // First CopyToReg
3167 TCChain = UseChain;
3168 }
3169 }
3170 } else if (Copy->getOpcode() == ISD::BITCAST) {
3171 // f32 returned in a single GPR.
3172 if (!Copy->hasOneUse())
3173 return false;
3174 Copy = *Copy->user_begin();
3175 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0))
3176 return false;
3177 // If the copy has a glue operand, we conservatively assume it isn't safe to
3178 // perform a tail call.
3179 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3180 return false;
3181 TCChain = Copy->getOperand(0);
3182 } else {
3183 return false;
3184 }
3185
3186 bool HasRet = false;
3187 for (const SDNode *U : Copy->users()) {
3188 if (U->getOpcode() != ARMISD::RET_GLUE &&
3189 U->getOpcode() != ARMISD::INTRET_GLUE)
3190 return false;
3191 HasRet = true;
3192 }
3193
3194 if (!HasRet)
3195 return false;
3196
3197 Chain = TCChain;
3198 return true;
3199}
3200
3201bool ARMTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
3202 if (!Subtarget->supportsTailCall())
3203 return false;
3204
3205 if (!CI->isTailCall())
3206 return false;
3207
3208 return true;
3209}
3210
3211// Trying to write a 64 bit value so need to split into two 32 bit values first,
3212// and pass the lower and high parts through.
3214 SDLoc DL(Op);
3215 SDValue WriteValue = Op->getOperand(2);
3216
3217 // This function is only supposed to be called for i64 type argument.
3218 assert(WriteValue.getValueType() == MVT::i64
3219 && "LowerWRITE_REGISTER called for non-i64 type argument.");
3220
3221 SDValue Lo, Hi;
3222 std::tie(Lo, Hi) = DAG.SplitScalar(WriteValue, DL, MVT::i32, MVT::i32);
3223 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi };
3224 return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops);
3225}
3226
3227// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
3228// their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
3229// one of the above mentioned nodes. It has to be wrapped because otherwise
3230// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
3231// be used to form addressing mode. These wrapped nodes will be selected
3232// into MOVi.
3233SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
3234 SelectionDAG &DAG) const {
3235 EVT PtrVT = Op.getValueType();
3236 // FIXME there is no actual debug info here
3237 SDLoc dl(Op);
3238 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
3239 SDValue Res;
3240
3241 // When generating execute-only code Constant Pools must be promoted to the
3242 // global data section. It's a bit ugly that we can't share them across basic
3243 // blocks, but this way we guarantee that execute-only behaves correct with
3244 // position-independent addressing modes.
3245 if (Subtarget->genExecuteOnly()) {
3246 auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
3247 auto *T = CP->getType();
3248 auto C = const_cast<Constant*>(CP->getConstVal());
3249 auto M = DAG.getMachineFunction().getFunction().getParent();
3250 auto GV = new GlobalVariable(
3251 *M, T, /*isConstant=*/true, GlobalVariable::InternalLinkage, C,
3252 Twine(DAG.getDataLayout().getInternalSymbolPrefix()) + "CP" +
3253 Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" +
3254 Twine(AFI->createPICLabelUId()));
3255 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
3256 return LowerGlobalAddress(GA, DAG);
3257 }
3258
3259 // The 16-bit ADR instruction can only encode offsets that are multiples of 4,
3260 // so we need to align to at least 4 bytes when we don't have 32-bit ADR.
3261 Align CPAlign = CP->getAlign();
3262 if (Subtarget->isThumb1Only())
3263 CPAlign = std::max(CPAlign, Align(4));
3265 Res =
3266 DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CPAlign);
3267 else
3268 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CPAlign);
3269 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res);
3270}
3271
3273 // If we don't have a 32-bit pc-relative branch instruction then the jump
3274 // table consists of block addresses. Usually this is inline, but for
3275 // execute-only it must be placed out-of-line.
3276 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3279}
3280
3281SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
3282 SelectionDAG &DAG) const {
3285 unsigned ARMPCLabelIndex = 0;
3286 SDLoc DL(Op);
3287 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3288 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
3289 SDValue CPAddr;
3290 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI();
3291 if (!IsPositionIndependent) {
3292 CPAddr = DAG.getTargetConstantPool(BA, PtrVT, Align(4));
3293 } else {
3294 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3295 ARMPCLabelIndex = AFI->createPICLabelUId();
3297 ARMConstantPoolConstant::Create(BA, ARMPCLabelIndex,
3298 ARMCP::CPBlockAddress, PCAdj);
3299 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3300 }
3301 CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr);
3302 SDValue Result = DAG.getLoad(
3303 PtrVT, DL, DAG.getEntryNode(), CPAddr,
3305 if (!IsPositionIndependent)
3306 return Result;
3307 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
3308 return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel);
3309}
3310
3311/// Convert a TLS address reference into the correct sequence of loads
3312/// and calls to compute the variable's address for Darwin, and return an
3313/// SDValue containing the final node.
3314
3315/// Darwin only has one TLS scheme which must be capable of dealing with the
3316/// fully general situation, in the worst case. This means:
3317/// + "extern __thread" declaration.
3318/// + Defined in a possibly unknown dynamic library.
3319///
3320/// The general system is that each __thread variable has a [3 x i32] descriptor
3321/// which contains information used by the runtime to calculate the address. The
3322/// only part of this the compiler needs to know about is the first word, which
3323/// contains a function pointer that must be called with the address of the
3324/// entire descriptor in "r0".
3325///
3326/// Since this descriptor may be in a different unit, in general access must
3327/// proceed along the usual ARM rules. A common sequence to produce is:
3328///
3329/// movw rT1, :lower16:_var$non_lazy_ptr
3330/// movt rT1, :upper16:_var$non_lazy_ptr
3331/// ldr r0, [rT1]
3332/// ldr rT2, [r0]
3333/// blx rT2
3334/// [...address now in r0...]
3335SDValue
3336ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
3337 SelectionDAG &DAG) const {
3338 assert(getTargetMachine().getTargetTriple().isOSDarwin() &&
3339 "This function expects a Darwin target");
3340 SDLoc DL(Op);
3341
3342 // First step is to get the address of the actua global symbol. This is where
3343 // the TLS descriptor lives.
3344 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
3345
3346 // The first entry in the descriptor is a function pointer that we must call
3347 // to obtain the address of the variable.
3348 SDValue Chain = DAG.getEntryNode();
3349 SDValue FuncTLVGet = DAG.getLoad(
3350 MVT::i32, DL, Chain, DescAddr,
3354 Chain = FuncTLVGet.getValue(1);
3355
3357 MachineFrameInfo &MFI = F.getFrameInfo();
3358 MFI.setAdjustsStack(true);
3359
3360 // TLS calls preserve all registers except those that absolutely must be
3361 // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be
3362 // silly).
3363 auto TRI =
3365 auto ARI = static_cast<const ARMRegisterInfo *>(TRI);
3366 const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction());
3367
3368 // Finally, we can make the call. This is just a degenerate version of a
3369 // normal AArch64 call node: r0 takes the address of the descriptor, and
3370 // returns the address of the variable in this thread.
3371 Chain = DAG.getCopyToReg(Chain, DL, ARM::R0, DescAddr, SDValue());
3372 Chain =
3373 DAG.getNode(ARMISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
3374 Chain, FuncTLVGet, DAG.getRegister(ARM::R0, MVT::i32),
3375 DAG.getRegisterMask(Mask), Chain.getValue(1));
3376 return DAG.getCopyFromReg(Chain, DL, ARM::R0, MVT::i32, Chain.getValue(1));
3377}
3378
3379SDValue
3380ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
3381 SelectionDAG &DAG) const {
3382 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
3383 "Windows specific TLS lowering");
3384
3385 SDValue Chain = DAG.getEntryNode();
3386 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3387 SDLoc DL(Op);
3388
3389 // Load the current TEB (thread environment block)
3390 SDValue Ops[] = {Chain,
3391 DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
3392 DAG.getTargetConstant(15, DL, MVT::i32),
3393 DAG.getTargetConstant(0, DL, MVT::i32),
3394 DAG.getTargetConstant(13, DL, MVT::i32),
3395 DAG.getTargetConstant(0, DL, MVT::i32),
3396 DAG.getTargetConstant(2, DL, MVT::i32)};
3397 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
3398 DAG.getVTList(MVT::i32, MVT::Other), Ops);
3399
3400 SDValue TEB = CurrentTEB.getValue(0);
3401 Chain = CurrentTEB.getValue(1);
3402
3403 // Load the ThreadLocalStoragePointer from the TEB
3404 // A pointer to the TLS array is located at offset 0x2c from the TEB.
3405 SDValue TLSArray =
3406 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x2c, DL));
3407 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
3408
3409 // The pointer to the thread's TLS data area is at the TLS Index scaled by 4
3410 // offset into the TLSArray.
3411
3412 // Load the TLS index from the C runtime
3413 SDValue TLSIndex =
3414 DAG.getTargetExternalSymbol("_tls_index", PtrVT, ARMII::MO_NO_FLAG);
3415 TLSIndex = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, TLSIndex);
3416 TLSIndex = DAG.getLoad(PtrVT, DL, Chain, TLSIndex, MachinePointerInfo());
3417
3418 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
3419 DAG.getConstant(2, DL, MVT::i32));
3420 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
3421 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
3422 MachinePointerInfo());
3423
3424 // Get the offset of the start of the .tls section (section base)
3425 const auto *GA = cast<GlobalAddressSDNode>(Op);
3426 auto *CPV = ARMConstantPoolConstant::Create(GA->getGlobal(), ARMCP::SECREL);
3427 SDValue Offset = DAG.getLoad(
3428 PtrVT, DL, Chain,
3429 DAG.getNode(ARMISD::Wrapper, DL, MVT::i32,
3430 DAG.getTargetConstantPool(CPV, PtrVT, Align(4))),
3432
3433 return DAG.getNode(ISD::ADD, DL, PtrVT, TLS, Offset);
3434}
3435
3436// Lower ISD::GlobalTLSAddress using the "general dynamic" model
3437SDValue
3438ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
3439 SelectionDAG &DAG) const {
3440 SDLoc dl(GA);
3441 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3442 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3444 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3445 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3446 ARMConstantPoolValue *CPV =
3447 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
3448 ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true);
3449 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3450 Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
3451 Argument = DAG.getLoad(
3452 PtrVT, dl, DAG.getEntryNode(), Argument,
3454 SDValue Chain = Argument.getValue(1);
3455
3456 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3457 Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
3458
3459 // call __tls_get_addr.
3461 Args.emplace_back(Argument, Type::getInt32Ty(*DAG.getContext()));
3462
3463 // FIXME: is there useful debug info available here?
3464 TargetLowering::CallLoweringInfo CLI(DAG);
3465 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3467 DAG.getExternalSymbol("__tls_get_addr", PtrVT), std::move(Args));
3468
3469 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3470 return CallResult.first;
3471}
3472
3473// Lower ISD::GlobalTLSAddress using the "initial exec" or
3474// "local exec" model.
3475SDValue
3476ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
3477 SelectionDAG &DAG,
3478 TLSModel::Model model) const {
3479 const GlobalValue *GV = GA->getGlobal();
3480 SDLoc dl(GA);
3482 SDValue Chain = DAG.getEntryNode();
3483 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3484 // Get the Thread Pointer
3485 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3486
3487 if (model == TLSModel::InitialExec) {
3489 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3490 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3491 // Initial exec model.
3492 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3493 ARMConstantPoolValue *CPV =
3494 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
3496 true);
3497 Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3498 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
3499 Offset = DAG.getLoad(
3500 PtrVT, dl, Chain, Offset,
3502 Chain = Offset.getValue(1);
3503
3504 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3505 Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
3506
3507 Offset = DAG.getLoad(
3508 PtrVT, dl, Chain, Offset,
3510 } else {
3511 // local exec model
3512 assert(model == TLSModel::LocalExec);
3513 ARMConstantPoolValue *CPV =
3515 Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3516 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
3517 Offset = DAG.getLoad(
3518 PtrVT, dl, Chain, Offset,
3520 }
3521
3522 // The address of the thread local variable is the add of the thread
3523 // pointer with the offset of the variable.
3524 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
3525}
3526
3527SDValue
3528ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
3529 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
3530 if (DAG.getTarget().useEmulatedTLS())
3531 return LowerToTLSEmulatedModel(GA, DAG);
3532
3533 const Triple &TT = getTargetMachine().getTargetTriple();
3534 if (TT.isOSDarwin())
3535 return LowerGlobalTLSAddressDarwin(Op, DAG);
3536
3537 if (TT.isOSWindows())
3538 return LowerGlobalTLSAddressWindows(Op, DAG);
3539
3540 // TODO: implement the "local dynamic" model
3541 assert(TT.isOSBinFormatELF() && "Only ELF implemented here");
3543
3544 switch (model) {
3547 return LowerToTLSGeneralDynamicModel(GA, DAG);
3550 return LowerToTLSExecModels(GA, DAG, model);
3551 }
3552 llvm_unreachable("bogus TLS model");
3553}
3554
3555/// Return true if all users of V are within function F, looking through
3556/// ConstantExprs.
3557static bool allUsersAreInFunction(const Value *V, const Function *F) {
3558 SmallVector<const User*,4> Worklist(V->users());
3559 while (!Worklist.empty()) {
3560 auto *U = Worklist.pop_back_val();
3561 if (isa<ConstantExpr>(U)) {
3562 append_range(Worklist, U->users());
3563 continue;
3564 }
3565
3566 auto *I = dyn_cast<Instruction>(U);
3567 if (!I || I->getParent()->getParent() != F)
3568 return false;
3569 }
3570 return true;
3571}
3572
3574 const GlobalValue *GV, SelectionDAG &DAG,
3575 EVT PtrVT, const SDLoc &dl) {
3576 // If we're creating a pool entry for a constant global with unnamed address,
3577 // and the global is small enough, we can emit it inline into the constant pool
3578 // to save ourselves an indirection.
3579 //
3580 // This is a win if the constant is only used in one function (so it doesn't
3581 // need to be duplicated) or duplicating the constant wouldn't increase code
3582 // size (implying the constant is no larger than 4 bytes).
3583 const Function &F = DAG.getMachineFunction().getFunction();
3584
3585 // We rely on this decision to inline being idempotent and unrelated to the
3586 // use-site. We know that if we inline a variable at one use site, we'll
3587 // inline it elsewhere too (and reuse the constant pool entry). Fast-isel
3588 // doesn't know about this optimization, so bail out if it's enabled else
3589 // we could decide to inline here (and thus never emit the GV) but require
3590 // the GV from fast-isel generated code.
3593 return SDValue();
3594
3595 auto *GVar = dyn_cast<GlobalVariable>(GV);
3596 if (!GVar || !GVar->hasInitializer() ||
3597 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3598 !GVar->hasLocalLinkage())
3599 return SDValue();
3600
3601 // If we inline a value that contains relocations, we move the relocations
3602 // from .data to .text. This is not allowed in position-independent code.
3603 auto *Init = GVar->getInitializer();
3604 if ((TLI->isPositionIndependent() || TLI->getSubtarget()->isROPI()) &&
3605 Init->needsDynamicRelocation())
3606 return SDValue();
3607
3608 // The constant islands pass can only really deal with alignment requests
3609 // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote
3610 // any type wanting greater alignment requirements than 4 bytes. We also
3611 // can only promote constants that are multiples of 4 bytes in size or
3612 // are paddable to a multiple of 4. Currently we only try and pad constants
3613 // that are strings for simplicity.
3614 auto *CDAInit = dyn_cast<ConstantDataArray>(Init);
3615 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType());
3616 Align PrefAlign = DAG.getDataLayout().getPreferredAlign(GVar);
3617 unsigned RequiredPadding = 4 - (Size % 4);
3618 bool PaddingPossible =
3619 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3620 if (!PaddingPossible || PrefAlign > 4 || Size > ConstpoolPromotionMaxSize ||
3621 Size == 0)
3622 return SDValue();
3623
3624 unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3626 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3627
3628 // We can't bloat the constant pool too much, else the ConstantIslands pass
3629 // may fail to converge. If we haven't promoted this global yet (it may have
3630 // multiple uses), and promoting it would increase the constant pool size (Sz
3631 // > 4), ensure we have space to do so up to MaxTotal.
3632 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar) && Size > 4)
3633 if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >=
3635 return SDValue();
3636
3637 // This is only valid if all users are in a single function; we can't clone
3638 // the constant in general. The LLVM IR unnamed_addr allows merging
3639 // constants, but not cloning them.
3640 //
3641 // We could potentially allow cloning if we could prove all uses of the
3642 // constant in the current function don't care about the address, like
3643 // printf format strings. But that isn't implemented for now.
3644 if (!allUsersAreInFunction(GVar, &F))
3645 return SDValue();
3646
3647 // We're going to inline this global. Pad it out if needed.
3648 if (RequiredPadding != 4) {
3649 StringRef S = CDAInit->getAsString();
3650
3652 std::copy(S.bytes_begin(), S.bytes_end(), V.begin());
3653 while (RequiredPadding--)
3654 V.push_back(0);
3656 }
3657
3658 auto CPVal = ARMConstantPoolConstant::Create(GVar, Init);
3659 SDValue CPAddr = DAG.getTargetConstantPool(CPVal, PtrVT, Align(4));
3660 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar)) {
3663 PaddedSize - 4);
3664 }
3665 ++NumConstpoolPromoted;
3666 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3667}
3668
3670 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3671 if (!(GV = GA->getAliaseeObject()))
3672 return false;
3673 if (const auto *V = dyn_cast<GlobalVariable>(GV))
3674 return V->isConstant();
3675 return isa<Function>(GV);
3676}
3677
3678SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
3679 SelectionDAG &DAG) const {
3680 switch (Subtarget->getTargetTriple().getObjectFormat()) {
3681 default: llvm_unreachable("unknown object format");
3682 case Triple::COFF:
3683 return LowerGlobalAddressWindows(Op, DAG);
3684 case Triple::ELF:
3685 return LowerGlobalAddressELF(Op, DAG);
3686 case Triple::MachO:
3687 return LowerGlobalAddressDarwin(Op, DAG);
3688 }
3689}
3690
3691SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
3692 SelectionDAG &DAG) const {
3693 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3694 SDLoc dl(Op);
3695 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3696 bool IsRO = isReadOnly(GV);
3697
3698 // promoteToConstantPool only if not generating XO text section
3699 if (GV->isDSOLocal() && !Subtarget->genExecuteOnly())
3700 if (SDValue V = promoteToConstantPool(this, GV, DAG, PtrVT, dl))
3701 return V;
3702
3703 if (isPositionIndependent()) {
3705 GV, dl, PtrVT, 0, GV->isDSOLocal() ? 0 : ARMII::MO_GOT);
3706 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3707 if (!GV->isDSOLocal())
3708 Result =
3709 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
3711 return Result;
3712 } else if (Subtarget->isROPI() && IsRO) {
3713 // PC-relative.
3714 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
3715 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3716 return Result;
3717 } else if (Subtarget->isRWPI() && !IsRO) {
3718 // SB-relative.
3719 SDValue RelAddr;
3720 if (Subtarget->useMovt()) {
3721 ++NumMovwMovt;
3722 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL);
3723 RelAddr = DAG.getNode(ARMISD::Wrapper, dl, PtrVT, G);
3724 } else { // use literal pool for address constant
3725 ARMConstantPoolValue *CPV =
3727 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3728 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3729 RelAddr = DAG.getLoad(
3730 PtrVT, dl, DAG.getEntryNode(), CPAddr,
3732 }
3733 SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT);
3734 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr);
3735 return Result;
3736 }
3737
3738 // If we have T2 ops, we can materialize the address directly via movt/movw
3739 // pair. This is always cheaper. If need to generate Execute Only code, and we
3740 // only have Thumb1 available, we can't use a constant pool and are forced to
3741 // use immediate relocations.
3742 if (Subtarget->useMovt() || Subtarget->genExecuteOnly()) {
3743 if (Subtarget->useMovt())
3744 ++NumMovwMovt;
3745 // FIXME: Once remat is capable of dealing with instructions with register
3746 // operands, expand this into two nodes.
3747 return DAG.getNode(ARMISD::Wrapper, dl, PtrVT,
3748 DAG.getTargetGlobalAddress(GV, dl, PtrVT));
3749 } else {
3750 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, Align(4));
3751 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3752 return DAG.getLoad(
3753 PtrVT, dl, DAG.getEntryNode(), CPAddr,
3755 }
3756}
3757
3758SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
3759 SelectionDAG &DAG) const {
3760 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3761 "ROPI/RWPI not currently supported for Darwin");
3762 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3763 SDLoc dl(Op);
3764 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3765
3766 if (Subtarget->useMovt())
3767 ++NumMovwMovt;
3768
3769 // FIXME: Once remat is capable of dealing with instructions with register
3770 // operands, expand this into multiple nodes
3771 unsigned Wrapper =
3772 isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper;
3773
3774 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY);
3775 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G);
3776
3777 if (Subtarget->isGVIndirectSymbol(GV))
3778 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
3780 return Result;
3781}
3782
3783SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3784 SelectionDAG &DAG) const {
3785 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
3786 "non-Windows COFF is not supported");
3787 assert(Subtarget->useMovt() &&
3788 "Windows on ARM expects to use movw/movt");
3789 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3790 "ROPI/RWPI not currently supported for Windows");
3791
3792 const TargetMachine &TM = getTargetMachine();
3793 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3794 ARMII::TOF TargetFlags = ARMII::MO_NO_FLAG;
3795 if (GV->hasDLLImportStorageClass())
3796 TargetFlags = ARMII::MO_DLLIMPORT;
3797 else if (!TM.shouldAssumeDSOLocal(GV))
3798 TargetFlags = ARMII::MO_COFFSTUB;
3799 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3801 SDLoc DL(Op);
3802
3803 ++NumMovwMovt;
3804
3805 // FIXME: Once remat is capable of dealing with instructions with register
3806 // operands, expand this into two nodes.
3807 Result = DAG.getNode(ARMISD::Wrapper, DL, PtrVT,
3808 DAG.getTargetGlobalAddress(GV, DL, PtrVT, /*offset=*/0,
3809 TargetFlags));
3810 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
3811 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
3813 return Result;
3814}
3815
3816SDValue
3817ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
3818 SDLoc dl(Op);
3819 SDValue Val = DAG.getConstant(0, dl, MVT::i32);
3820 return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl,
3821 DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
3822 Op.getOperand(1), Val);
3823}
3824
3825SDValue
3826ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
3827 SDLoc dl(Op);
3828 return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0),
3829 Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32));
3830}
3831
3832SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
3833 SelectionDAG &DAG) const {
3834 SDLoc dl(Op);
3835 return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other,
3836 Op.getOperand(0));
3837}
3838
3839SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
3840 SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
3841 unsigned IntNo =
3842 Op.getConstantOperandVal(Op.getOperand(0).getValueType() == MVT::Other);
3843 switch (IntNo) {
3844 default:
3845 return SDValue(); // Don't custom lower most intrinsics.
3846 case Intrinsic::arm_gnu_eabi_mcount: {
3848 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3849 SDLoc dl(Op);
3850 SDValue Chain = Op.getOperand(0);
3851 // call "\01__gnu_mcount_nc"
3852 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
3853 const uint32_t *Mask =
3855 assert(Mask && "Missing call preserved mask for calling convention");
3856 // Mark LR an implicit live-in.
3857 Register Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
3858 SDValue ReturnAddress =
3859 DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, PtrVT);
3860 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
3861 SDValue Callee =
3862 DAG.getTargetExternalSymbol("\01__gnu_mcount_nc", PtrVT, 0);
3864 if (Subtarget->isThumb())
3865 return SDValue(
3866 DAG.getMachineNode(
3867 ARM::tBL_PUSHLR, dl, ResultTys,
3868 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
3869 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
3870 0);
3871 return SDValue(
3872 DAG.getMachineNode(ARM::BL_PUSHLR, dl, ResultTys,
3873 {ReturnAddress, Callee, RegisterMask, Chain}),
3874 0);
3875 }
3876 }
3877}
3878
3879SDValue
3880ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
3881 const ARMSubtarget *Subtarget) const {
3882 unsigned IntNo = Op.getConstantOperandVal(0);
3883 SDLoc dl(Op);
3884 switch (IntNo) {
3885 default: return SDValue(); // Don't custom lower most intrinsics.
3886 case Intrinsic::localaddress: {
3887 const MachineFunction &MF = DAG.getMachineFunction();
3888 const auto *RegInfo = Subtarget->getRegisterInfo();
3889 unsigned Reg = RegInfo->getLocalAddressRegister(MF);
3890 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg,
3891 Op.getSimpleValueType());
3892 }
3893 case Intrinsic::eh_recoverfp: {
3894 SDValue FnOp = Op.getOperand(1);
3895 GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
3896 auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
3897 if (!Fn)
3899 "llvm.eh.recoverfp must take a function as the first argument");
3900 const auto *RegInfo = Subtarget->getRegisterInfo();
3901 Register BaseReg = RegInfo->getBaseRegister();
3903 MachineBasicBlock &MBB = *MF.begin();
3904 if (!MBB.isLiveIn(BaseReg))
3905 MBB.addLiveIn(BaseReg);
3906 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3907 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, BaseReg, PtrVT);
3908 }
3909 case Intrinsic::thread_pointer: {
3910 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3911 return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3912 }
3913 case Intrinsic::arm_cls: {
3914 // Note: arm_cls and arm_cls64 intrinsics are expanded directly here
3915 // in LowerINTRINSIC_WO_CHAIN since there's no native scalar CLS
3916 // instruction.
3917 const SDValue &Operand = Op.getOperand(1);
3918 const EVT VTy = Op.getValueType();
3919 return DAG.getNode(ISD::CTLS, dl, VTy, Operand);
3920 }
3921 case Intrinsic::arm_cls64: {
3922 // arm_cls64 returns i32 but takes i64 input.
3923 // Use ISD::CTLS for i64 and truncate the result.
3924 SDValue CTLS64 = DAG.getNode(ISD::CTLS, dl, MVT::i64, Op.getOperand(1));
3925 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, CTLS64);
3926 }
3927 case Intrinsic::arm_neon_vcls:
3928 case Intrinsic::arm_mve_vcls: {
3929 // Lower vector CLS intrinsics to ISD::CTLS.
3930 // Vector CTLS is Legal when NEON/MVE is available (set elsewhere).
3931 const EVT VTy = Op.getValueType();
3932 return DAG.getNode(ISD::CTLS, dl, VTy, Op.getOperand(1));
3933 }
3934 case Intrinsic::eh_sjlj_lsda: {
3936 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3937 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3938 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3939 SDValue CPAddr;
3940 bool IsPositionIndependent = isPositionIndependent();
3941 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
3942 ARMConstantPoolValue *CPV =
3943 ARMConstantPoolConstant::Create(&MF.getFunction(), ARMPCLabelIndex,
3944 ARMCP::CPLSDA, PCAdj);
3945 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3946 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3947 SDValue Result = DAG.getLoad(
3948 PtrVT, dl, DAG.getEntryNode(), CPAddr,
3950
3951 if (IsPositionIndependent) {
3952 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3953 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
3954 }
3955 return Result;
3956 }
3957 case Intrinsic::arm_neon_vabs:
3958 return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(),
3959 Op.getOperand(1));
3960 case Intrinsic::arm_neon_vabds:
3961 if (Op.getValueType().isInteger())
3962 return DAG.getNode(ISD::ABDS, SDLoc(Op), Op.getValueType(),
3963 Op.getOperand(1), Op.getOperand(2));
3964 return SDValue();
3965 case Intrinsic::arm_neon_vabdu:
3966 return DAG.getNode(ISD::ABDU, SDLoc(Op), Op.getValueType(),
3967 Op.getOperand(1), Op.getOperand(2));
3968 case Intrinsic::arm_neon_vmulls:
3969 case Intrinsic::arm_neon_vmullu: {
3970 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
3971 ? ARMISD::VMULLs : ARMISD::VMULLu;
3972 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3973 Op.getOperand(1), Op.getOperand(2));
3974 }
3975 case Intrinsic::arm_neon_vminnm:
3976 case Intrinsic::arm_neon_vmaxnm: {
3977 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
3978 ? ISD::FMINNUM : ISD::FMAXNUM;
3979 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3980 Op.getOperand(1), Op.getOperand(2));
3981 }
3982 case Intrinsic::arm_neon_vminu:
3983 case Intrinsic::arm_neon_vmaxu: {
3984 if (Op.getValueType().isFloatingPoint())
3985 return SDValue();
3986 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
3987 ? ISD::UMIN : ISD::UMAX;
3988 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3989 Op.getOperand(1), Op.getOperand(2));
3990 }
3991 case Intrinsic::arm_neon_vmins:
3992 case Intrinsic::arm_neon_vmaxs: {
3993 // v{min,max}s is overloaded between signed integers and floats.
3994 if (!Op.getValueType().isFloatingPoint()) {
3995 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3996 ? ISD::SMIN : ISD::SMAX;
3997 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3998 Op.getOperand(1), Op.getOperand(2));
3999 }
4000 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4001 ? ISD::FMINIMUM : ISD::FMAXIMUM;
4002 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4003 Op.getOperand(1), Op.getOperand(2));
4004 }
4005 case Intrinsic::arm_neon_vtbl1:
4006 return DAG.getNode(ARMISD::VTBL1, SDLoc(Op), Op.getValueType(),
4007 Op.getOperand(1), Op.getOperand(2));
4008 case Intrinsic::arm_neon_vtbl2:
4009 return DAG.getNode(ARMISD::VTBL2, SDLoc(Op), Op.getValueType(),
4010 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4011 case Intrinsic::arm_mve_pred_i2v:
4012 case Intrinsic::arm_mve_pred_v2i:
4013 return DAG.getNode(ARMISD::PREDICATE_CAST, SDLoc(Op), Op.getValueType(),
4014 Op.getOperand(1));
4015 case Intrinsic::arm_mve_vreinterpretq:
4016 return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(Op), Op.getValueType(),
4017 Op.getOperand(1));
4018 case Intrinsic::arm_mve_lsll:
4019 return DAG.getNode(ARMISD::LSLL, SDLoc(Op), Op->getVTList(),
4020 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4021 case Intrinsic::arm_mve_asrl:
4022 return DAG.getNode(ARMISD::ASRL, SDLoc(Op), Op->getVTList(),
4023 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4024 case Intrinsic::arm_mve_vsli:
4025 return DAG.getNode(ARMISD::VSLIIMM, SDLoc(Op), Op->getVTList(),
4026 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4027 case Intrinsic::arm_mve_vsri:
4028 return DAG.getNode(ARMISD::VSRIIMM, SDLoc(Op), Op->getVTList(),
4029 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4030 }
4031}
4032
4034 const ARMSubtarget *Subtarget) {
4035 SDLoc dl(Op);
4036 auto SSID = static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
4037 if (SSID == SyncScope::SingleThread)
4038 return Op;
4039
4040 if (!Subtarget->hasDataBarrier()) {
4041 // Some ARMv6 cpus can support data barriers with an mcr instruction.
4042 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
4043 // here.
4044 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4045 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4046 return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
4047 DAG.getConstant(0, dl, MVT::i32));
4048 }
4049
4050 AtomicOrdering Ord =
4051 static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
4053 if (Subtarget->isMClass()) {
4054 // Only a full system barrier exists in the M-class architectures.
4056 } else if (Subtarget->preferISHSTBarriers() &&
4057 Ord == AtomicOrdering::Release) {
4058 // Swift happens to implement ISHST barriers in a way that's compatible with
4059 // Release semantics but weaker than ISH so we'd be fools not to use
4060 // it. Beware: other processors probably don't!
4062 }
4063
4064 return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0),
4065 DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4066 DAG.getConstant(Domain, dl, MVT::i32));
4067}
4068
4070 const ARMSubtarget *Subtarget) {
4071 // ARM pre v5TE and Thumb1 does not have preload instructions.
4072 if (!(Subtarget->isThumb2() ||
4073 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps())))
4074 // Just preserve the chain.
4075 return Op.getOperand(0);
4076
4077 SDLoc dl(Op);
4078 unsigned isRead = ~Op.getConstantOperandVal(2) & 1;
4079 if (!isRead &&
4080 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4081 // ARMv7 with MP extension has PLDW.
4082 return Op.getOperand(0);
4083
4084 unsigned isData = Op.getConstantOperandVal(4);
4085 if (Subtarget->isThumb()) {
4086 // Invert the bits.
4087 isRead = ~isRead & 1;
4088 isData = ~isData & 1;
4089 }
4090
4091 return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0),
4092 Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32),
4093 DAG.getConstant(isData, dl, MVT::i32));
4094}
4095
4098 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>();
4099
4100 // vastart just stores the address of the VarArgsFrameIndex slot into the
4101 // memory location argument.
4102 SDLoc dl(Op);
4104 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4105 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4106 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
4107 MachinePointerInfo(SV));
4108}
4109
4110SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
4111 CCValAssign &NextVA,
4112 SDValue &Root,
4113 SelectionDAG &DAG,
4114 const SDLoc &dl) const {
4116 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4117
4118 const TargetRegisterClass *RC;
4119 if (AFI->isThumb1OnlyFunction())
4120 RC = &ARM::tGPRRegClass;
4121 else
4122 RC = &ARM::GPRRegClass;
4123
4124 // Transform the arguments stored in physical registers into virtual ones.
4125 Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
4126 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
4127
4128 SDValue ArgValue2;
4129 if (NextVA.isMemLoc()) {
4130 MachineFrameInfo &MFI = MF.getFrameInfo();
4131 int FI = MFI.CreateFixedObject(4, NextVA.getLocMemOffset(), true);
4132
4133 // Create load node to retrieve arguments from the stack.
4134 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4135 ArgValue2 = DAG.getLoad(
4136 MVT::i32, dl, Root, FIN,
4138 } else {
4139 Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
4140 ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
4141 }
4142 if (!Subtarget->isLittle())
4143 std::swap (ArgValue, ArgValue2);
4144 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2);
4145}
4146
4147// The remaining GPRs hold either the beginning of variable-argument
4148// data, or the beginning of an aggregate passed by value (usually
4149// byval). Either way, we allocate stack slots adjacent to the data
4150// provided by our caller, and store the unallocated registers there.
4151// If this is a variadic function, the va_list pointer will begin with
4152// these values; otherwise, this reassembles a (byval) structure that
4153// was split between registers and memory.
4154// Return: The frame index registers were stored into.
4155int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
4156 const SDLoc &dl, SDValue &Chain,
4157 const Value *OrigArg,
4158 unsigned InRegsParamRecordIdx,
4159 int ArgOffset, unsigned ArgSize) const {
4160 // Currently, two use-cases possible:
4161 // Case #1. Non-var-args function, and we meet first byval parameter.
4162 // Setup first unallocated register as first byval register;
4163 // eat all remained registers
4164 // (these two actions are performed by HandleByVal method).
4165 // Then, here, we initialize stack frame with
4166 // "store-reg" instructions.
4167 // Case #2. Var-args function, that doesn't contain byval parameters.
4168 // The same: eat all remained unallocated registers,
4169 // initialize stack frame.
4170
4172 MachineFrameInfo &MFI = MF.getFrameInfo();
4173 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4174 unsigned RBegin, REnd;
4175 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) {
4176 CCInfo.getInRegsParamInfo(InRegsParamRecordIdx, RBegin, REnd);
4177 } else {
4178 unsigned RBeginIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
4179 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx];
4180 REnd = ARM::R4;
4181 }
4182
4183 if (REnd != RBegin)
4184 ArgOffset = -4 * (ARM::R4 - RBegin);
4185
4186 auto PtrVT = getPointerTy(DAG.getDataLayout());
4187 int FrameIndex = MFI.CreateFixedObject(ArgSize, ArgOffset, false);
4188 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
4189
4191 const TargetRegisterClass *RC =
4192 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
4193
4194 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) {
4195 Register VReg = MF.addLiveIn(Reg, RC);
4196 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
4197 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
4198 MachinePointerInfo(OrigArg, 4 * i));
4199 MemOps.push_back(Store);
4200 FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT));
4201 }
4202
4203 if (!MemOps.empty())
4204 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
4205 return FrameIndex;
4206}
4207
4208// Setup stack frame, the va_list pointer will start from.
4209void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
4210 const SDLoc &dl, SDValue &Chain,
4211 unsigned ArgOffset,
4212 unsigned TotalArgRegsSaveSize,
4213 bool ForceMutable) const {
4215 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4216
4217 // Try to store any remaining integer argument regs
4218 // to their spots on the stack so that they may be loaded by dereferencing
4219 // the result of va_next.
4220 // If there is no regs to be stored, just point address after last
4221 // argument passed via stack.
4222 int FrameIndex = StoreByValRegs(
4223 CCInfo, DAG, dl, Chain, nullptr, CCInfo.getInRegsParamsCount(),
4224 CCInfo.getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4225 AFI->setVarArgsFrameIndex(FrameIndex);
4226}
4227
4228bool ARMTargetLowering::splitValueIntoRegisterParts(
4229 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4230 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
4231 EVT ValueVT = Val.getValueType();
4232 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4233 unsigned ValueBits = ValueVT.getSizeInBits();
4234 unsigned PartBits = PartVT.getSizeInBits();
4235 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(ValueBits), Val);
4236 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::getIntegerVT(PartBits), Val);
4237 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
4238 Parts[0] = Val;
4239 return true;
4240 }
4241 return false;
4242}
4243
4244SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4245 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
4246 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
4247 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4248 unsigned ValueBits = ValueVT.getSizeInBits();
4249 unsigned PartBits = PartVT.getSizeInBits();
4250 SDValue Val = Parts[0];
4251
4252 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(PartBits), Val);
4253 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::getIntegerVT(ValueBits), Val);
4254 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
4255 return Val;
4256 }
4257 return SDValue();
4258}
4259
4260SDValue ARMTargetLowering::LowerFormalArguments(
4261 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4262 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4263 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4265 MachineFrameInfo &MFI = MF.getFrameInfo();
4266
4267 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4268
4269 // Assign locations to all of the incoming arguments.
4271 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4272 *DAG.getContext());
4273 CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForCall(CallConv, isVarArg));
4274
4276 unsigned CurArgIdx = 0;
4277
4278 // Initially ArgRegsSaveSize is zero.
4279 // Then we increase this value each time we meet byval parameter.
4280 // We also increase this value in case of varargs function.
4281 AFI->setArgRegsSaveSize(0);
4282
4283 // Calculate the amount of stack space that we need to allocate to store
4284 // byval and variadic arguments that are passed in registers.
4285 // We need to know this before we allocate the first byval or variadic
4286 // argument, as they will be allocated a stack slot below the CFA (Canonical
4287 // Frame Address, the stack pointer at entry to the function).
4288 unsigned ArgRegBegin = ARM::R4;
4289 for (const CCValAssign &VA : ArgLocs) {
4290 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount())
4291 break;
4292
4293 unsigned Index = VA.getValNo();
4294 ISD::ArgFlagsTy Flags = Ins[Index].Flags;
4295 if (!Flags.isByVal())
4296 continue;
4297
4298 assert(VA.isMemLoc() && "unexpected byval pointer in reg");
4299 unsigned RBegin, REnd;
4300 CCInfo.getInRegsParamInfo(CCInfo.getInRegsParamsProcessed(), RBegin, REnd);
4301 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4302
4303 CCInfo.nextInRegsParam();
4304 }
4305 CCInfo.rewindByValRegsInfo();
4306
4307 int lastInsIndex = -1;
4308 if (isVarArg && MFI.hasVAStart()) {
4309 unsigned RegIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
4310 if (RegIdx != std::size(GPRArgRegs))
4311 ArgRegBegin = std::min(ArgRegBegin, (unsigned)GPRArgRegs[RegIdx]);
4312 }
4313
4314 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4315 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
4316 auto PtrVT = getPointerTy(DAG.getDataLayout());
4317
4318 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4319 CCValAssign &VA = ArgLocs[i];
4320 if (Ins[VA.getValNo()].isOrigArg()) {
4321 std::advance(CurOrigArg,
4322 Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx);
4323 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex();
4324 }
4325 // Arguments stored in registers.
4326 if (VA.isRegLoc()) {
4327 EVT RegVT = VA.getLocVT();
4328 SDValue ArgValue;
4329
4330 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
4331 // f64 and vector types are split up into multiple registers or
4332 // combinations of registers and stack slots.
4333 SDValue ArgValue1 =
4334 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4335 VA = ArgLocs[++i]; // skip ahead to next loc
4336 SDValue ArgValue2;
4337 if (VA.isMemLoc()) {
4338 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), true);
4339 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4340 ArgValue2 = DAG.getLoad(
4341 MVT::f64, dl, Chain, FIN,
4343 } else {
4344 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4345 }
4346 ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
4347 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
4348 ArgValue1, DAG.getIntPtrConstant(0, dl));
4349 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
4350 ArgValue2, DAG.getIntPtrConstant(1, dl));
4351 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
4352 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4353 } else {
4354 const TargetRegisterClass *RC;
4355
4356 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4357 RC = &ARM::HPRRegClass;
4358 else if (RegVT == MVT::f32)
4359 RC = &ARM::SPRRegClass;
4360 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4361 RegVT == MVT::v4bf16)
4362 RC = &ARM::DPRRegClass;
4363 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4364 RegVT == MVT::v8bf16)
4365 RC = &ARM::QPRRegClass;
4366 else if (RegVT == MVT::i32)
4367 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass
4368 : &ARM::GPRRegClass;
4369 else
4370 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
4371
4372 // Transform the arguments in physical registers into virtual ones.
4373 Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
4374 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
4375
4376 // If this value is passed in r0 and has the returned attribute (e.g.
4377 // C++ 'structors), record this fact for later use.
4378 if (VA.getLocReg() == ARM::R0 && Ins[VA.getValNo()].Flags.isReturned()) {
4379 AFI->setPreservesR0();
4380 }
4381 }
4382
4383 // If this is an 8 or 16-bit value, it is really passed promoted
4384 // to 32 bits. Insert an assert[sz]ext to capture this, then
4385 // truncate to the right size.
4386 switch (VA.getLocInfo()) {
4387 default: llvm_unreachable("Unknown loc info!");
4388 case CCValAssign::Full: break;
4389 case CCValAssign::BCvt:
4390 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
4391 break;
4392 }
4393
4394 // f16 arguments have their size extended to 4 bytes and passed as if they
4395 // had been copied to the LSBs of a 32-bit register.
4396 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
4397 if (VA.needsCustom() &&
4398 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
4399 ArgValue = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), ArgValue);
4400
4401 // On CMSE Entry Functions, formal integer arguments whose bitwidth is
4402 // less than 32 bits must be sign- or zero-extended in the callee for
4403 // security reasons. Although the ABI mandates an extension done by the
4404 // caller, the latter cannot be trusted to follow the rules of the ABI.
4405 const ISD::InputArg &Arg = Ins[VA.getValNo()];
4406 if (AFI->isCmseNSEntryFunction() && Arg.ArgVT.isScalarInteger() &&
4407 RegVT.isScalarInteger() && Arg.ArgVT.bitsLT(MVT::i32))
4408 ArgValue = handleCMSEValue(ArgValue, Arg, DAG, dl);
4409
4410 InVals.push_back(ArgValue);
4411 } else { // VA.isRegLoc()
4412 // Only arguments passed on the stack should make it here.
4413 assert(VA.isMemLoc());
4414 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered");
4415
4416 int index = VA.getValNo();
4417
4418 // Some Ins[] entries become multiple ArgLoc[] entries.
4419 // Process them only once.
4420 if (index != lastInsIndex)
4421 {
4422 ISD::ArgFlagsTy Flags = Ins[index].Flags;
4423 // FIXME: For now, all byval parameter objects are marked mutable.
4424 // This can be changed with more analysis.
4425 // In case of tail call optimization mark all arguments mutable.
4426 // Since they could be overwritten by lowering of arguments in case of
4427 // a tail call.
4428 if (Flags.isByVal()) {
4429 assert(Ins[index].isOrigArg() &&
4430 "Byval arguments cannot be implicit");
4431 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed();
4432
4433 int FrameIndex = StoreByValRegs(
4434 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4435 VA.getLocMemOffset(), Flags.getByValSize());
4436 InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT));
4437 CCInfo.nextInRegsParam();
4438 } else if (VA.needsCustom() && (VA.getValVT() == MVT::f16 ||
4439 VA.getValVT() == MVT::bf16)) {
4440 // f16 and bf16 values are passed in the least-significant half of
4441 // a 4 byte stack slot. This is done as-if the extension was done
4442 // in a 32-bit register, so the actual bytes used for the value
4443 // differ between little and big endian.
4444 assert(VA.getLocVT().getSizeInBits() == 32);
4445 unsigned FIOffset = VA.getLocMemOffset();
4446 int FI = MFI.CreateFixedObject(VA.getLocVT().getSizeInBits() / 8,
4447 FIOffset, true);
4448
4449 SDValue Addr = DAG.getFrameIndex(FI, PtrVT);
4450 if (DAG.getDataLayout().isBigEndian())
4451 Addr = DAG.getObjectPtrOffset(dl, Addr, TypeSize::getFixed(2));
4452
4453 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, Addr,
4455 DAG.getMachineFunction(), FI)));
4456
4457 } else {
4458 unsigned FIOffset = VA.getLocMemOffset();
4459 int FI = MFI.CreateFixedObject(VA.getLocVT().getSizeInBits()/8,
4460 FIOffset, true);
4461
4462 // Create load nodes to retrieve arguments from the stack.
4463 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4464 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
4466 DAG.getMachineFunction(), FI)));
4467 }
4468 lastInsIndex = index;
4469 }
4470 }
4471 }
4472
4473 // varargs
4474 if (isVarArg && MFI.hasVAStart()) {
4475 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.getStackSize(),
4476 TotalArgRegsSaveSize);
4477 if (AFI->isCmseNSEntryFunction()) {
4478 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
4480 "secure entry function must not be variadic", dl.getDebugLoc()));
4481 }
4482 }
4483
4484 // A variadic function that forwards through a musttail call needs its unnamed
4485 // register arguments modelled as live, so the allocator does not use one as
4486 // scratch. This must run after the va_list save area above, which consumes
4487 // the same registers, and is deliberately *not* predicated on hasVAStart():
4488 // a forwarding thunk never calls va_start.
4489 if (isVarArg && MFI.hasMustTailInVarArgFunc()) {
4490 // getRemainingRegParmsForType() walks the assign function until an argument
4491 // lands in memory, so the convention must have a stack fallback. GHC and
4492 // the Windows CFGuard check stub assign to registers only and would run off
4493 // the end of their tables; neither supports variadic arguments anyway.
4494 CallingConv::ID EffCC = getEffectiveCallingConv(CallConv, isVarArg);
4495 if (EffCC != CallingConv::GHC && EffCC != CallingConv::CFGuard_Check) {
4496 // Only the core registers can hold an unnamed argument:
4497 // getEffectiveCallingConv() degrades every VFP convention to ARM_AAPCS
4498 // when isVarArg is set.
4499 SmallVector<MVT, 1> RegParmTypes;
4500 RegParmTypes.push_back(MVT::i32);
4501 SmallVectorImpl<ForwardedRegister> &Forwards =
4504 Forwards, RegParmTypes, CCAssignFnForCall(CallConv, isVarArg));
4505 }
4506 }
4507
4508 unsigned StackArgSize = CCInfo.getStackSize();
4509 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4510 if (canGuaranteeTCO(CallConv, TailCallOpt)) {
4511 // The only way to guarantee a tail call is if the callee restores its
4512 // argument area, but it must also keep the stack aligned when doing so.
4513 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
4514 assert(StackAlign && "data layout string is missing stack alignment");
4515 StackArgSize = alignTo(StackArgSize, *StackAlign);
4516
4517 AFI->setArgumentStackToRestore(StackArgSize);
4518 }
4519 AFI->setArgumentStackSize(StackArgSize);
4520
4521 if (CCInfo.getStackSize() > 0 && AFI->isCmseNSEntryFunction()) {
4522 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
4524 "secure entry function requires arguments on stack", dl.getDebugLoc()));
4525 }
4526
4527 return Chain;
4528}
4529
4530/// isFloatingPointZero - Return true if this is +0.0.
4533 return CFP->getValueAPF().isPosZero();
4534 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
4535 // Maybe this has already been legalized into the constant pool?
4536 if (Op.getOperand(1).getOpcode() == ARMISD::Wrapper) {
4537 SDValue WrapperOp = Op.getOperand(1).getOperand(0);
4539 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4540 return CFP->getValueAPF().isPosZero();
4541 }
4542 } else if (Op->getOpcode() == ISD::BITCAST &&
4543 Op->getValueType(0) == MVT::f64) {
4544 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64)
4545 // created by LowerConstantFP().
4546 SDValue BitcastOp = Op->getOperand(0);
4547 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM &&
4548 isNullConstant(BitcastOp->getOperand(0)))
4549 return true;
4550 }
4551 return false;
4552}
4553
4555 // 0 - INT_MIN sign wraps, so no signed wrap means cmn is safe.
4556 if (Op->getFlags().hasNoSignedWrap())
4557 return true;
4558
4559 // We can still figure out if the second operand is safe to use
4560 // in a CMN instruction by checking if it is known to be not the minimum
4561 // signed value. If it is not, then we can safely use CMN.
4562 // Note: We can eventually remove this check and simply rely on
4563 // Op->getFlags().hasNoSignedWrap() once SelectionDAG/ISelLowering
4564 // consistently sets them appropriately when making said nodes.
4565
4566 KnownBits KnownSrc = DAG.computeKnownBits(Op.getOperand(1));
4567 return !KnownSrc.getSignedMinValue().isMinSignedValue();
4568}
4569
4571 return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
4572 (isIntEqualitySetCC(CC) ||
4573 (isUnsignedIntSetCC(CC) && DAG.isKnownNeverZero(Op.getOperand(1))) ||
4574 (isSignedIntSetCC(CC) && isSafeSignedCMN(Op, DAG)));
4575}
4576
4577/// Returns how profitable it is to fold a comparison's operand's shift and/or
4578/// extension operations into the comparison instruction's second operand
4579/// (so_reg_imm / so_reg_reg for ARM, t2_so_reg for Thumb-2).
4581 // Thumb-1 CMP does not support shifted second operands.
4582 if (ST.isThumb1Only() || !Op.hasOneUse())
4583 return 0;
4584
4585 unsigned Opc = Op.getOpcode();
4586 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) {
4587 if (auto *ShiftAmt = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
4588 return ShiftAmt->getZExtValue() <= 31 ? 1 : 0;
4589 // Register-controlled shift: only ARM-mode CMP/CMN (so_reg_reg) supports
4590 // this; Thumb-2 t2_so_reg requires an immediate shift amount.
4591 return ST.isThumb() ? 0 : 1;
4592 }
4593
4594 if (Opc == ISD::ROTR) {
4595 // Rotr constants will be normalized via mod 32, or & 31,
4596 // so we do not have to bounds check.
4597 if (isa<ConstantSDNode>(Op.getOperand(1)))
4598 return 1;
4599 return ST.isThumb() ? 0 : 1;
4600 }
4601
4602 return 0;
4603}
4604
4605// Return a cheaper value with the same zero/nonzero result as a low-mask AND.
4607 const SDLoc &DL) {
4608 if (Value.getValueType() != MVT::i32 || Value.getOpcode() != ISD::AND ||
4609 !Value.hasOneUse())
4610 return SDValue();
4611
4612 auto *Mask = dyn_cast<ConstantSDNode>(Value.getOperand(1));
4613 if (!Mask)
4614 return SDValue();
4615 uint32_t MaskValue = Mask->getZExtValue();
4616 // Keep the existing single-bit lowering and UXTB/UXTH for byte/halfword
4617 // masks, and avoid a zero or no-op shift.
4618 if (!isMask_32(MaskValue) || MaskValue == 1 || MaskValue == 0xff ||
4619 MaskValue == 0xffff || MaskValue == ~0U)
4620 return SDValue();
4621
4622 return DAG.getNode(
4623 ISD::SHL, DL, MVT::i32, Value.getOperand(0),
4624 DAG.getConstant(llvm::countl_zero(MaskValue), DL, MVT::i32));
4625}
4626
4627/// Returns appropriate ARM CMP (cmp) and corresponding condition code for
4628/// the given operands.
4629SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4630 SDValue &ARMcc, SelectionDAG &DAG,
4631 const SDLoc &dl) const {
4632 // For a zero test, replace a low-mask AND feeding an OR with a shift. ARM
4633 // and Thumb-2 can fold the shift into the OR. Require single uses to avoid
4634 // duplicating live work.
4635 if (!Subtarget->isThumb1Only() && isIntEqualitySetCC(CC) &&
4636 isNullConstant(RHS) && LHS.getValueType() == MVT::i32 &&
4637 LHS.getOpcode() == ISD::OR && LHS.hasOneUse()) {
4638 for (unsigned I = 0; I != 2; ++I) {
4639 if (SDValue Shift =
4640 getShiftedLowMaskZeroTest(LHS.getOperand(I), DAG, dl)) {
4641 LHS = DAG.getNode(ISD::OR, dl, MVT::i32, LHS.getOperand(I ^ 1), Shift);
4642 break;
4643 }
4644 }
4645 }
4646
4647 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
4648 unsigned C = RHSC->getZExtValue();
4649 if (!isLegalICmpImmediate((int32_t)C)) {
4650 // Constant does not fit, try adjusting it by one.
4651 switch (CC) {
4652 default: break;
4653 case ISD::SETLT:
4654 case ISD::SETGE:
4655 if (C != 0x80000000 && isLegalICmpImmediate(C-1)) {
4656 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
4657 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
4658 }
4659 break;
4660 case ISD::SETULT:
4661 case ISD::SETUGE:
4662 if (C != 0 && isLegalICmpImmediate(C-1)) {
4663 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
4664 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
4665 }
4666 break;
4667 case ISD::SETLE:
4668 case ISD::SETGT:
4669 if (C != 0x7fffffff && isLegalICmpImmediate(C+1)) {
4670 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
4671 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
4672 }
4673 break;
4674 case ISD::SETULE:
4675 case ISD::SETUGT:
4676 if (C != 0xffffffff && isLegalICmpImmediate(C+1)) {
4677 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4678 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
4679 }
4680 break;
4681 }
4682 }
4683 }
4684
4685 // Thumb1 has very limited immediate modes, so turning an "and" into a
4686 // shift can save multiple instructions.
4687 //
4688 // If we have (x & C1), and C1 is an appropriate mask, we can transform it
4689 // into "((x << n) >> n)". But that isn't necessarily profitable on its
4690 // own. If it's the operand to an unsigned comparison with an immediate,
4691 // we can eliminate one of the shifts: we transform
4692 // "((x << n) >> n) == C2" to "(x << n) == (C2 << n)".
4693 //
4694 // We avoid transforming cases which aren't profitable due to encoding
4695 // details:
4696 //
4697 // 1. C2 fits into the immediate field of a cmp, and the transformed version
4698 // would not; in that case, we're essentially trading one immediate load for
4699 // another.
4700 // 2. C1 is 255 or 65535, so we can use uxtb or uxth.
4701 // 3. C2 is zero; we have other code for this special case.
4702 //
4703 // FIXME: Figure out profitability for Thumb2; we usually can't save an
4704 // instruction, since the AND is always one instruction anyway, but we could
4705 // use narrow instructions in some cases.
4706 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::AND &&
4707 LHS->hasOneUse() && isa<ConstantSDNode>(LHS.getOperand(1)) &&
4708 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4709 !isSignedIntSetCC(CC)) {
4710 unsigned Mask = LHS.getConstantOperandVal(1);
4711 auto *RHSC = cast<ConstantSDNode>(RHS.getNode());
4712 uint64_t RHSV = RHSC->getZExtValue();
4713 if (isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4714 unsigned ShiftBits = llvm::countl_zero(Mask);
4715 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4716 SDValue ShiftAmt = DAG.getConstant(ShiftBits, dl, MVT::i32);
4717 LHS = DAG.getNode(ISD::SHL, dl, MVT::i32, LHS.getOperand(0), ShiftAmt);
4718 RHS = DAG.getConstant(RHSV << ShiftBits, dl, MVT::i32);
4719 }
4720 }
4721 }
4722
4723 // The specific comparison "(x<<c) > 0x80000000U" can be optimized to a
4724 // single "lsls x, c+1". The shift sets the "C" and "Z" flags the same
4725 // way a cmp would.
4726 // FIXME: Add support for ARM/Thumb2; this would need isel patterns, and
4727 // some tweaks to the heuristics for the previous and->shift transform.
4728 // FIXME: Optimize cases where the LHS isn't a shift.
4729 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::SHL &&
4730 isa<ConstantSDNode>(RHS) && RHS->getAsZExtVal() == 0x80000000U &&
4731 CC == ISD::SETUGT && isa<ConstantSDNode>(LHS.getOperand(1)) &&
4732 LHS.getConstantOperandVal(1) < 31) {
4733 unsigned ShiftAmt = LHS.getConstantOperandVal(1) + 1;
4734 SDValue Shift =
4735 DAG.getNode(ARMISD::LSLS, dl, DAG.getVTList(MVT::i32, FlagsVT),
4736 LHS.getOperand(0), DAG.getConstant(ShiftAmt, dl, MVT::i32));
4737 ARMcc = DAG.getConstant(ARMCC::HI, dl, MVT::i32);
4738 return Shift.getValue(1);
4739 }
4740
4742
4743 unsigned CompareType;
4744 switch (CondCode) {
4745 default:
4746 CompareType = ARMISD::CMP;
4747 break;
4748 case ARMCC::EQ:
4749 case ARMCC::NE:
4750 // Uses only Z Flag
4751 CompareType = ARMISD::CMPZ;
4752 break;
4753 }
4754
4755 // TODO: Remove CMPZ check once we generalize and remove the CMPZ enum from
4756 // the codebase.
4757
4758 // TODO: When we have a solution to the vselect predicate not allowing pl/mi
4759 // all the time, allow those cases to be cmn too no matter what.
4760 if (CompareType != ARMISD::CMPZ && isCMN(RHS, CC, DAG)) {
4761 CompareType = ARMISD::CMN;
4762 RHS = RHS.getOperand(1);
4763 } else if (CompareType != ARMISD::CMPZ && isCMN(LHS, CC, DAG)) {
4764 CompareType = ARMISD::CMN;
4765 LHS = LHS.getOperand(1);
4767 }
4768
4769 // Prefer folding shifts / CMN into the cmp/cmn second operand (so_reg /
4770 // t2_so_reg). When both sides compete, pick the higher
4771 // getCmpOperandFoldingProfit. Only when RHS is not a legal icmp
4772 // immediate: otherwise keep the canonical (reg, imm) form.
4773 ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS.getNode());
4774 if (!C || !isLegalICmpImmediate(C->getSExtValue())) {
4775 if (getCmpOperandFoldingProfit(LHS, *Subtarget) >
4776 getCmpOperandFoldingProfit(RHS, *Subtarget)) {
4777 std::swap(LHS, RHS);
4778 if (CompareType == ARMISD::CMP)
4780 }
4781 }
4782
4783 // If the RHS is a constant zero then the V (overflow) flag will never be
4784 // set. This can allow us to simplify GE to PL or LT to MI, which can be
4785 // simpler for other passes (like the peephole optimiser) to deal with.
4786 if (isNullConstant(RHS)) {
4787 switch (CondCode) {
4788 default:
4789 break;
4790 case ARMCC::GE:
4792 break;
4793 case ARMCC::LT:
4795 break;
4796 }
4797 }
4798
4799 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
4800 return DAG.getNode(CompareType, dl, FlagsVT, LHS, RHS);
4801}
4802
4803/// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
4804SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4805 SelectionDAG &DAG, const SDLoc &dl,
4806 bool Signaling) const {
4807 assert(Subtarget->hasFP64() || RHS.getValueType() != MVT::f64);
4808 SDValue Flags;
4810 Flags = DAG.getNode(Signaling ? ARMISD::CMPFPE : ARMISD::CMPFP, dl, FlagsVT,
4811 LHS, RHS);
4812 else
4813 Flags = DAG.getNode(Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0, dl,
4814 FlagsVT, LHS);
4815 return DAG.getNode(ARMISD::FMSTAT, dl, FlagsVT, Flags);
4816}
4817
4818// This function returns three things: the arithmetic computation itself
4819// (Value), a comparison (OverflowCmp), and a condition code (ARMcc). The
4820// comparison and the condition code define the case in which the arithmetic
4821// computation *does not* overflow.
4822std::pair<SDValue, SDValue>
4823ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
4824 SDValue &ARMcc) const {
4825 assert(Op.getValueType() == MVT::i32 && "Unsupported value type");
4826
4827 SDValue Value, OverflowCmp;
4828 SDValue LHS = Op.getOperand(0);
4829 SDValue RHS = Op.getOperand(1);
4830 SDLoc dl(Op);
4831
4832 // FIXME: We are currently always generating CMPs because we don't support
4833 // generating CMN through the backend. This is not as good as the natural
4834 // CMP case because it causes a register dependency and cannot be folded
4835 // later.
4836
4837 switch (Op.getOpcode()) {
4838 default:
4839 llvm_unreachable("Unknown overflow instruction!");
4840 case ISD::SADDO:
4841 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
4842 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
4843 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value, LHS);
4844 break;
4845 case ISD::UADDO:
4846 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
4847 // We use ADDC here to correspond to its use in LowerALUO.
4848 // We do not use it in the USUBO case as Value may not be used.
4849 Value = DAG.getNode(ARMISD::ADDC, dl,
4850 DAG.getVTList(Op.getValueType(), MVT::i32), LHS, RHS)
4851 .getValue(0);
4852 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value, LHS);
4853 break;
4854 case ISD::SSUBO:
4855 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
4856 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
4857 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, LHS, RHS);
4858 break;
4859 case ISD::USUBO:
4860 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
4861 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
4862 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, LHS, RHS);
4863 break;
4864 case ISD::UMULO:
4865 // We generate a UMUL_LOHI and then check if the high word is 0.
4866 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
4867 Value = DAG.getNode(ISD::UMUL_LOHI, dl,
4868 DAG.getVTList(Op.getValueType(), Op.getValueType()),
4869 LHS, RHS);
4870 OverflowCmp = DAG.getNode(ARMISD::CMPZ, dl, FlagsVT, Value.getValue(1),
4871 DAG.getConstant(0, dl, MVT::i32));
4872 Value = Value.getValue(0); // We only want the low 32 bits for the result.
4873 break;
4874 case ISD::SMULO:
4875 // We generate a SMUL_LOHI and then check if all the bits of the high word
4876 // are the same as the sign bit of the low word.
4877 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
4878 Value = DAG.getNode(ISD::SMUL_LOHI, dl,
4879 DAG.getVTList(Op.getValueType(), Op.getValueType()),
4880 LHS, RHS);
4881 OverflowCmp = DAG.getNode(ARMISD::CMPZ, dl, FlagsVT, Value.getValue(1),
4882 DAG.getNode(ISD::SRA, dl, Op.getValueType(),
4883 Value.getValue(0),
4884 DAG.getConstant(31, dl, MVT::i32)));
4885 Value = Value.getValue(0); // We only want the low 32 bits for the result.
4886 break;
4887 } // switch (...)
4888
4889 return std::make_pair(Value, OverflowCmp);
4890}
4891
4893 SDLoc DL(Value);
4894 EVT VT = Value.getValueType();
4895
4896 if (Invert)
4897 Value = DAG.getNode(ISD::SUB, DL, MVT::i32,
4898 DAG.getConstant(1, DL, MVT::i32), Value);
4899
4900 SDValue Cmp = DAG.getNode(ARMISD::SUBC, DL, DAG.getVTList(VT, MVT::i32),
4901 Value, DAG.getConstant(1, DL, VT));
4902 return Cmp.getValue(1);
4903}
4904
4906 bool Invert) {
4907 SDLoc DL(Flags);
4908
4909 if (Invert) {
4910 // Convert flags to boolean with ADDE 0,0,Carry then compute 1 - bool.
4911 SDValue BoolCarry = DAG.getNode(
4912 ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32),
4913 DAG.getConstant(0, DL, VT), DAG.getConstant(0, DL, VT), Flags);
4914 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(1, DL, VT), BoolCarry);
4915 }
4916
4917 // Now convert the carry flag into a boolean carry. We do this
4918 // using ARMISD::ADDE 0, 0, Carry
4919 return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32),
4920 DAG.getConstant(0, DL, VT), DAG.getConstant(0, DL, VT),
4921 Flags);
4922}
4923
4924// Value is 1 if 'V' bit is 1, else 0
4926 SDLoc DL(Flags);
4927 SDValue Zero = DAG.getConstant(0, DL, VT);
4928 SDValue One = DAG.getConstant(1, DL, VT);
4929 SDValue ARMcc = DAG.getConstant(ARMCC::VS, DL, MVT::i32);
4930 return DAG.getNode(ARMISD::CMOV, DL, VT, Zero, One, ARMcc, Flags);
4931}
4932
4933SDValue ARMTargetLowering::LowerALUO(SDValue Op, SelectionDAG &DAG) const {
4934 // Let legalize expand this if it isn't a legal type yet.
4935 if (!isTypeLegal(Op.getValueType()))
4936 return SDValue();
4937
4938 SDValue LHS = Op.getOperand(0);
4939 SDValue RHS = Op.getOperand(1);
4940 SDLoc dl(Op);
4941
4942 EVT VT = Op.getValueType();
4943 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
4944 SDValue Value;
4945 SDValue Overflow;
4946 switch (Op.getOpcode()) {
4947 case ISD::UADDO:
4948 Value = DAG.getNode(ARMISD::ADDC, dl, VTs, LHS, RHS);
4949 // Convert the carry flag into a boolean value.
4950 Overflow = carryFlagToValue(Value.getValue(1), VT, DAG, false);
4951 break;
4952 case ISD::USUBO:
4953 Value = DAG.getNode(ARMISD::SUBC, dl, VTs, LHS, RHS);
4954 // Convert the carry flag into a boolean value.
4955 Overflow = carryFlagToValue(Value.getValue(1), VT, DAG, true);
4956 break;
4957 default: {
4958 // Handle other operations with getARMXALUOOp
4959 SDValue OverflowCmp, ARMcc;
4960 std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
4961 // We use 0 and 1 as false and true values.
4962 // ARMcc represents the "no overflow" condition (e.g., VC for signed ops).
4963 // CMOV operand order is (FalseVal, TrueVal), so we put 1 in FalseVal
4964 // position to get Overflow=1 when the "no overflow" condition is false.
4965 Overflow =
4966 DAG.getNode(ARMISD::CMOV, dl, MVT::i32,
4967 DAG.getConstant(1, dl, MVT::i32), // FalseVal: overflow
4968 DAG.getConstant(0, dl, MVT::i32), // TrueVal: no overflow
4969 ARMcc, OverflowCmp);
4970 break;
4971 }
4972 }
4973
4974 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
4975}
4976
4978 const ARMSubtarget *Subtarget) {
4979 EVT VT = Op.getValueType();
4980 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->isThumb1Only())
4981 return SDValue();
4982 if (!VT.isSimple())
4983 return SDValue();
4984
4985 unsigned NewOpcode;
4986 switch (VT.getSimpleVT().SimpleTy) {
4987 default:
4988 return SDValue();
4989 case MVT::i8:
4990 switch (Op->getOpcode()) {
4991 case ISD::UADDSAT:
4992 NewOpcode = ARMISD::UQADD8b;
4993 break;
4994 case ISD::SADDSAT:
4995 NewOpcode = ARMISD::QADD8b;
4996 break;
4997 case ISD::USUBSAT:
4998 NewOpcode = ARMISD::UQSUB8b;
4999 break;
5000 case ISD::SSUBSAT:
5001 NewOpcode = ARMISD::QSUB8b;
5002 break;
5003 }
5004 break;
5005 case MVT::i16:
5006 switch (Op->getOpcode()) {
5007 case ISD::UADDSAT:
5008 NewOpcode = ARMISD::UQADD16b;
5009 break;
5010 case ISD::SADDSAT:
5011 NewOpcode = ARMISD::QADD16b;
5012 break;
5013 case ISD::USUBSAT:
5014 NewOpcode = ARMISD::UQSUB16b;
5015 break;
5016 case ISD::SSUBSAT:
5017 NewOpcode = ARMISD::QSUB16b;
5018 break;
5019 }
5020 break;
5021 }
5022
5023 SDLoc dl(Op);
5024 SDValue Add =
5025 DAG.getNode(NewOpcode, dl, MVT::i32,
5026 DAG.getSExtOrTrunc(Op->getOperand(0), dl, MVT::i32),
5027 DAG.getSExtOrTrunc(Op->getOperand(1), dl, MVT::i32));
5028 return DAG.getNode(ISD::TRUNCATE, dl, VT, Add);
5029}
5030
5031SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
5032 SDValue Cond = Op.getOperand(0);
5033 SDValue SelectTrue = Op.getOperand(1);
5034 SDValue SelectFalse = Op.getOperand(2);
5035 SDLoc dl(Op);
5036 unsigned Opc = Cond.getOpcode();
5037
5038 if (Cond.getResNo() == 1 &&
5039 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5040 Opc == ISD::USUBO)) {
5041 if (!isTypeLegal(Cond->getValueType(0)))
5042 return SDValue();
5043
5044 SDValue Value, OverflowCmp;
5045 SDValue ARMcc;
5046 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
5047 EVT VT = Op.getValueType();
5048
5049 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, OverflowCmp, DAG);
5050 }
5051
5052 // Convert:
5053 //
5054 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond)
5055 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond)
5056 //
5057 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) {
5058 const ConstantSDNode *CMOVTrue =
5059 dyn_cast<ConstantSDNode>(Cond.getOperand(0));
5060 const ConstantSDNode *CMOVFalse =
5061 dyn_cast<ConstantSDNode>(Cond.getOperand(1));
5062
5063 if (CMOVTrue && CMOVFalse) {
5064 unsigned CMOVTrueVal = CMOVTrue->getZExtValue();
5065 unsigned CMOVFalseVal = CMOVFalse->getZExtValue();
5066
5067 SDValue True;
5068 SDValue False;
5069 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5070 True = SelectTrue;
5071 False = SelectFalse;
5072 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5073 True = SelectFalse;
5074 False = SelectTrue;
5075 }
5076
5077 if (True.getNode() && False.getNode())
5078 return getCMOV(dl, Op.getValueType(), True, False, Cond.getOperand(2),
5079 Cond.getOperand(3), DAG);
5080 }
5081 }
5082
5083 return DAG.getSelectCC(dl, Cond,
5084 DAG.getConstant(0, dl, Cond.getValueType()),
5085 SelectTrue, SelectFalse, ISD::SETNE);
5086}
5087
5089 bool &swpCmpOps, bool &swpVselOps) {
5090 // Start by selecting the GE condition code for opcodes that return true for
5091 // 'equality'
5092 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE ||
5093 CC == ISD::SETULE || CC == ISD::SETGE || CC == ISD::SETLE)
5094 CondCode = ARMCC::GE;
5095
5096 // and GT for opcodes that return false for 'equality'.
5097 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT ||
5098 CC == ISD::SETULT || CC == ISD::SETGT || CC == ISD::SETLT)
5099 CondCode = ARMCC::GT;
5100
5101 // Since we are constrained to GE/GT, if the opcode contains 'less', we need
5102 // to swap the compare operands.
5103 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT ||
5104 CC == ISD::SETULT || CC == ISD::SETLE || CC == ISD::SETLT)
5105 swpCmpOps = true;
5106
5107 // Both GT and GE are ordered comparisons, and return false for 'unordered'.
5108 // If we have an unordered opcode, we need to swap the operands to the VSEL
5109 // instruction (effectively negating the condition).
5110 //
5111 // This also has the effect of swapping which one of 'less' or 'greater'
5112 // returns true, so we also swap the compare operands. It also switches
5113 // whether we return true for 'equality', so we compensate by picking the
5114 // opposite condition code to our original choice.
5115 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE ||
5116 CC == ISD::SETUGT) {
5117 swpCmpOps = !swpCmpOps;
5118 swpVselOps = !swpVselOps;
5119 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT;
5120 }
5121
5122 // 'ordered' is 'anything but unordered', so use the VS condition code and
5123 // swap the VSEL operands.
5124 if (CC == ISD::SETO) {
5125 CondCode = ARMCC::VS;
5126 swpVselOps = true;
5127 }
5128
5129 // 'unordered or not equal' is 'anything but equal', so use the EQ condition
5130 // code and swap the VSEL operands. Also do this if we don't care about the
5131 // unordered case.
5132 if (CC == ISD::SETUNE || CC == ISD::SETNE) {
5133 CondCode = ARMCC::EQ;
5134 swpVselOps = true;
5135 }
5136}
5137
5138SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
5139 SDValue TrueVal, SDValue ARMcc,
5140 SDValue Flags, SelectionDAG &DAG) const {
5141 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5142 FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl,
5143 DAG.getVTList(MVT::i32, MVT::i32), FalseVal);
5144 TrueVal = DAG.getNode(ARMISD::VMOVRRD, dl,
5145 DAG.getVTList(MVT::i32, MVT::i32), TrueVal);
5146
5147 SDValue TrueLow = TrueVal.getValue(0);
5148 SDValue TrueHigh = TrueVal.getValue(1);
5149 SDValue FalseLow = FalseVal.getValue(0);
5150 SDValue FalseHigh = FalseVal.getValue(1);
5151
5152 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
5153 ARMcc, Flags);
5154 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
5155 ARMcc, Flags);
5156
5157 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High);
5158 }
5159 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, Flags);
5160}
5161
5163 return CC == ISD::SETGT || CC == ISD::SETGE;
5164}
5165
5167 return CC == ISD::SETLT || CC == ISD::SETLE;
5168}
5169
5170// See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
5171// All of these conditions (and their <= and >= counterparts) will do:
5172// x < k ? k : x
5173// x > k ? x : k
5174// k < x ? x : k
5175// k > x ? k : x
5176static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
5177 const SDValue TrueVal, const SDValue FalseVal,
5178 const ISD::CondCode CC, const SDValue K) {
5179 return (isGTorGE(CC) &&
5180 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
5181 (isLTorLE(CC) &&
5182 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
5183}
5184
5185// Check if two chained conditionals could be converted into SSAT or USAT.
5186//
5187// SSAT can replace a set of two conditional selectors that bound a number to an
5188// interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples:
5189//
5190// x < -k ? -k : (x > k ? k : x)
5191// x < -k ? -k : (x < k ? x : k)
5192// x > -k ? (x > k ? k : x) : -k
5193// x < k ? (x < -k ? -k : x) : k
5194// etc.
5195//
5196// LLVM canonicalizes these to either a min(max()) or a max(min())
5197// pattern. This function tries to match one of these and will return a SSAT
5198// node if successful.
5199//
5200// USAT works similarly to SSAT but bounds on the interval [0, k] where k + 1
5201// is a power of 2.
5203 EVT VT = Op.getValueType();
5204 SDValue V1 = Op.getOperand(0);
5205 SDValue K1 = Op.getOperand(1);
5206 SDValue TrueVal1 = Op.getOperand(2);
5207 SDValue FalseVal1 = Op.getOperand(3);
5208 ISD::CondCode CC1 = Op.getOperandCondCode(4);
5209
5210 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5211 if (Op2.getOpcode() != ISD::SELECT_CC)
5212 return SDValue();
5213
5214 SDValue V2 = Op2.getOperand(0);
5215 SDValue K2 = Op2.getOperand(1);
5216 SDValue TrueVal2 = Op2.getOperand(2);
5217 SDValue FalseVal2 = Op2.getOperand(3);
5218 ISD::CondCode CC2 = Op2.getOperandCondCode(4);
5219
5220 SDValue V1Tmp = V1;
5221 SDValue V2Tmp = V2;
5222
5223 // Check that the registers and the constants match a max(min()) or min(max())
5224 // pattern
5225 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5226 K2 != FalseVal2 ||
5227 !((isGTorGE(CC1) && isLTorLE(CC2)) || (isLTorLE(CC1) && isGTorGE(CC2))))
5228 return SDValue();
5229
5230 // Check that the constant in the lower-bound check is
5231 // the opposite of the constant in the upper-bound check
5232 // in 1's complement.
5234 return SDValue();
5235
5236 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5237 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5238 int64_t PosVal = std::max(Val1, Val2);
5239 int64_t NegVal = std::min(Val1, Val2);
5240
5241 if (!((Val1 > Val2 && isLTorLE(CC1)) || (Val1 < Val2 && isLTorLE(CC2))) ||
5242 !isPowerOf2_64(PosVal + 1))
5243 return SDValue();
5244
5245 // Handle the difference between USAT (unsigned) and SSAT (signed)
5246 // saturation
5247 // At this point, PosVal is guaranteed to be positive
5248 uint64_t K = PosVal;
5249 SDLoc dl(Op);
5250 if (Val1 == ~Val2)
5251 return DAG.getNode(ARMISD::SSAT, dl, VT, V2Tmp,
5252 DAG.getConstant(llvm::countr_one(K), dl, VT));
5253 if (NegVal == 0)
5254 return DAG.getNode(ARMISD::USAT, dl, VT, V2Tmp,
5255 DAG.getConstant(llvm::countr_one(K), dl, VT));
5256
5257 return SDValue();
5258}
5259
5260// Check if a condition of the type x < k ? k : x can be converted into a
5261// bit operation instead of conditional moves.
5262// Currently this is allowed given:
5263// - The conditions and values match up
5264// - k is 0 or -1 (all ones)
5265// This function will not check the last condition, thats up to the caller
5266// It returns true if the transformation can be made, and in such case
5267// returns x in V, and k in SatK.
5269 SDValue &SatK)
5270{
5271 SDValue LHS = Op.getOperand(0);
5272 SDValue RHS = Op.getOperand(1);
5273 ISD::CondCode CC = Op.getOperandCondCode(4);
5274 SDValue TrueVal = Op.getOperand(2);
5275 SDValue FalseVal = Op.getOperand(3);
5276
5278 ? &RHS
5279 : nullptr;
5280
5281 // No constant operation in comparison, early out
5282 if (!K)
5283 return false;
5284
5285 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5286 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5287 SDValue VTmp = (K && *K == LHS) ? RHS : LHS;
5288
5289 // If the constant on left and right side, or variable on left and right,
5290 // does not match, early out
5291 if (*K != KTmp || V != VTmp)
5292 return false;
5293
5294 if (isLowerSaturate(LHS, RHS, TrueVal, FalseVal, CC, *K)) {
5295 SatK = *K;
5296 return true;
5297 }
5298
5299 return false;
5300}
5301
5302bool ARMTargetLowering::isUnsupportedFloatingType(EVT VT) const {
5303 if (VT == MVT::f32)
5304 return !Subtarget->hasVFP2Base();
5305 if (VT == MVT::f64)
5306 return !Subtarget->hasFP64();
5307 if (VT == MVT::f16)
5308 return !Subtarget->hasFullFP16();
5309 return false;
5310}
5311
5312static SDValue matchCSET(unsigned &Opcode, bool &InvertCond, SDValue TrueVal,
5313 SDValue FalseVal, const ARMSubtarget *Subtarget) {
5314 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
5315 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TrueVal);
5316 if (!CFVal || !CTVal || !Subtarget->hasV8_1MMainlineOps())
5317 return SDValue();
5318
5319 unsigned TVal = CTVal->getZExtValue();
5320 unsigned FVal = CFVal->getZExtValue();
5321
5322 Opcode = 0;
5323 InvertCond = false;
5324 if (TVal == ~FVal) {
5325 Opcode = ARMISD::CSINV;
5326 } else if (TVal == ~FVal + 1) {
5327 Opcode = ARMISD::CSNEG;
5328 } else if (TVal + 1 == FVal) {
5329 Opcode = ARMISD::CSINC;
5330 } else if (TVal == FVal + 1) {
5331 Opcode = ARMISD::CSINC;
5332 std::swap(TrueVal, FalseVal);
5333 std::swap(TVal, FVal);
5334 InvertCond = !InvertCond;
5335 } else {
5336 return SDValue();
5337 }
5338
5339 // If one of the constants is cheaper than another, materialise the
5340 // cheaper one and let the csel generate the other.
5341 if (Opcode != ARMISD::CSINC &&
5343 std::swap(TrueVal, FalseVal);
5344 std::swap(TVal, FVal);
5345 InvertCond = !InvertCond;
5346 }
5347
5348 // Attempt to use ZR checking TVal is 0, possibly inverting the condition
5349 // to get there. CSINC not is invertable like the other two (~(~a) == a,
5350 // -(-a) == a, but (a+1)+1 != a).
5351 if (FVal == 0 && Opcode != ARMISD::CSINC) {
5352 std::swap(TrueVal, FalseVal);
5353 std::swap(TVal, FVal);
5354 InvertCond = !InvertCond;
5355 }
5356
5357 return TrueVal;
5358}
5359
5360SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
5361 EVT VT = Op.getValueType();
5362 SDLoc dl(Op);
5363
5364 // Try to convert two saturating conditional selects into a single SSAT
5365 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2())
5366 if (SDValue SatValue = LowerSaturatingConditional(Op, DAG))
5367 return SatValue;
5368
5369 // Try to convert expressions of the form x < k ? k : x (and similar forms)
5370 // into more efficient bit operations, which is possible when k is 0 or -1
5371 // On ARM and Thumb-2 which have flexible operand 2 this will result in
5372 // single instructions. On Thumb the shift and the bit operation will be two
5373 // instructions.
5374 // Only allow this transformation on full-width (32-bit) operations
5375 SDValue LowerSatConstant;
5376 SDValue SatValue;
5377 if (VT == MVT::i32 &&
5378 isLowerSaturatingConditional(Op, SatValue, LowerSatConstant)) {
5379 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue,
5380 DAG.getConstant(31, dl, VT));
5381 if (isNullConstant(LowerSatConstant)) {
5382 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV,
5383 DAG.getAllOnesConstant(dl, VT));
5384 return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV);
5385 } else if (isAllOnesConstant(LowerSatConstant))
5386 return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV);
5387 }
5388
5389 SDValue LHS = Op.getOperand(0);
5390 SDValue RHS = Op.getOperand(1);
5391 ISD::CondCode CC = Op.getOperandCondCode(4);
5392 SDValue TrueVal = Op.getOperand(2);
5393 SDValue FalseVal = Op.getOperand(3);
5394 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
5395 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
5396 if (Op.getValueType().isInteger()) {
5397
5398 // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns.
5399 // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1))
5400 // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1))
5401 // Both require less instructions than compare and conditional select.
5402 if ((CC == ISD::SETGT || CC == ISD::SETLT) && LHS == TrueVal && RHSC &&
5403 RHSC->isZero() && CFVal && CFVal->isZero() &&
5404 LHS.getValueType() == RHS.getValueType()) {
5405 EVT VT = LHS.getValueType();
5406 SDValue Shift =
5407 DAG.getNode(ISD::SRA, dl, VT, LHS,
5408 DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
5409
5410 if (CC == ISD::SETGT)
5411 Shift = DAG.getNOT(dl, Shift, VT);
5412
5413 return DAG.getNode(ISD::AND, dl, VT, LHS, Shift);
5414 }
5415
5416 // (SELECT_CC setlt, x, 0, 1, 0) -> SRL(x, bw-1)
5417 if (CC == ISD::SETLT && isNullConstant(RHS) && isOneConstant(TrueVal) &&
5418 isNullConstant(FalseVal) && LHS.getValueType() == VT)
5419 return DAG.getNode(ISD::SRL, dl, VT, LHS,
5420 DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
5421 }
5422
5423 if (LHS.getValueType() == MVT::i32) {
5424 unsigned Opcode;
5425 bool InvertCond;
5426 if (SDValue Op =
5427 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
5428 if (InvertCond)
5429 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5430
5431 SDValue ARMcc;
5432 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5433 EVT VT = Op.getValueType();
5434 return DAG.getNode(Opcode, dl, VT, Op, Op, ARMcc, Cmp);
5435 }
5436 }
5437
5438 if (isUnsupportedFloatingType(LHS.getValueType())) {
5439 softenSetCCOperands(DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
5440
5441 // If softenSetCCOperands only returned one value, we should compare it to
5442 // zero.
5443 if (!RHS.getNode()) {
5444 RHS = DAG.getConstant(0, dl, LHS.getValueType());
5445 CC = ISD::SETNE;
5446 }
5447 }
5448
5449 if (LHS.getValueType() == MVT::i32) {
5450 // Try to generate VSEL on ARMv8.
5451 // The VSEL instruction can't use all the usual ARM condition
5452 // codes: it only has two bits to select the condition code, so it's
5453 // constrained to use only GE, GT, VS and EQ.
5454 //
5455 // To implement all the various ISD::SETXXX opcodes, we sometimes need to
5456 // swap the operands of the previous compare instruction (effectively
5457 // inverting the compare condition, swapping 'less' and 'greater') and
5458 // sometimes need to swap the operands to the VSEL (which inverts the
5459 // condition in the sense of firing whenever the previous condition didn't)
5460 if (Subtarget->hasFPARMv8Base() && (TrueVal.getValueType() == MVT::f16 ||
5461 TrueVal.getValueType() == MVT::f32 ||
5462 TrueVal.getValueType() == MVT::f64)) {
5464 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
5465 CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
5466 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5467 std::swap(TrueVal, FalseVal);
5468 }
5469 }
5470
5471 SDValue ARMcc;
5472 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5473 // Choose GE over PL, which vsel does now support
5474 if (ARMcc->getAsZExtVal() == ARMCC::PL)
5475 ARMcc = DAG.getConstant(ARMCC::GE, dl, MVT::i32);
5476 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
5477 }
5478
5479 ARMCC::CondCodes CondCode, CondCode2;
5480 FPCCToARMCC(CC, CondCode, CondCode2);
5481
5482 // Normalize the fp compare. If RHS is zero we prefer to keep it there so we
5483 // match CMPFPw0 instead of CMPFP, though we don't do this for f16 because we
5484 // must use VSEL (limited condition codes), due to not having conditional f16
5485 // moves.
5486 if (Subtarget->hasFPARMv8Base() &&
5487 !(isFloatingPointZero(RHS) && TrueVal.getValueType() != MVT::f16) &&
5488 (TrueVal.getValueType() == MVT::f16 ||
5489 TrueVal.getValueType() == MVT::f32 ||
5490 TrueVal.getValueType() == MVT::f64)) {
5491 bool swpCmpOps = false;
5492 bool swpVselOps = false;
5493 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps);
5494
5495 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE ||
5496 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) {
5497 if (swpCmpOps)
5498 std::swap(LHS, RHS);
5499 if (swpVselOps)
5500 std::swap(TrueVal, FalseVal);
5501 }
5502 }
5503
5504 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5505 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5506 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
5507 if (CondCode2 != ARMCC::AL) {
5508 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
5509 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, Cmp, DAG);
5510 }
5511 return Result;
5512}
5513
5514/// canChangeToInt - Given the fp compare operand, return true if it is suitable
5515/// to morph to an integer compare sequence.
5516static bool canChangeToInt(SDValue Op, bool &SeenZero,
5517 const ARMSubtarget *Subtarget) {
5518 SDNode *N = Op.getNode();
5519 if (!N->hasOneUse())
5520 // Otherwise it requires moving the value from fp to integer registers.
5521 return false;
5522 if (!N->getNumValues())
5523 return false;
5524 EVT VT = Op.getValueType();
5525 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5526 // f32 case is generally profitable. f64 case only makes sense when vcmpe +
5527 // vmrs are very slow, e.g. cortex-a8.
5528 return false;
5529
5530 if (isFloatingPointZero(Op)) {
5531 SeenZero = true;
5532 return true;
5533 }
5534 return ISD::isNormalLoad(N);
5535}
5536
5539 return DAG.getConstant(0, SDLoc(Op), MVT::i32);
5540
5542 return DAG.getLoad(MVT::i32, SDLoc(Op), Ld->getChain(), Ld->getBasePtr(),
5543 Ld->getPointerInfo(), Ld->getAlign(),
5544 Ld->getMemOperand()->getFlags());
5545
5546 llvm_unreachable("Unknown VFP cmp argument!");
5547}
5548
5550 SDValue &RetVal1, SDValue &RetVal2) {
5551 SDLoc dl(Op);
5552
5553 if (isFloatingPointZero(Op)) {
5554 RetVal1 = DAG.getConstant(0, dl, MVT::i32);
5555 RetVal2 = DAG.getConstant(0, dl, MVT::i32);
5556 return;
5557 }
5558
5559 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) {
5560 SDValue Ptr = Ld->getBasePtr();
5561 RetVal1 =
5562 DAG.getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(),
5563 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5564
5565 EVT PtrType = Ptr.getValueType();
5566 SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
5567 PtrType, Ptr, DAG.getConstant(4, dl, PtrType));
5568 RetVal2 = DAG.getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5569 Ld->getPointerInfo().getWithOffset(4),
5570 commonAlignment(Ld->getAlign(), 4),
5571 Ld->getMemOperand()->getFlags());
5572 return;
5573 }
5574
5575 llvm_unreachable("Unknown VFP cmp argument!");
5576}
5577
5578/// OptimizeVFPBrcond - With nnan and without daz, it's legal to optimize some
5579/// f32 and even f64 comparisons to integer ones.
5580SDValue
5581ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
5582 SDValue Chain = Op.getOperand(0);
5583 ISD::CondCode CC = Op.getOperandCondCode(1);
5584 SDValue LHS = Op.getOperand(2);
5585 SDValue RHS = Op.getOperand(3);
5586 SDValue Dest = Op.getOperand(4);
5587 SDLoc dl(Op);
5588
5589 bool LHSSeenZero = false;
5590 bool LHSOk = canChangeToInt(LHS, LHSSeenZero, Subtarget);
5591 bool RHSSeenZero = false;
5592 bool RHSOk = canChangeToInt(RHS, RHSSeenZero, Subtarget);
5593 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5594 // If unsafe fp math optimization is enabled and there are no other uses of
5595 // the CMP operands, and the condition code is EQ or NE, we can optimize it
5596 // to an integer comparison.
5597 if (CC == ISD::SETOEQ)
5598 CC = ISD::SETEQ;
5599 else if (CC == ISD::SETUNE)
5600 CC = ISD::SETNE;
5601
5602 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
5603 SDValue ARMcc;
5604 if (LHS.getValueType() == MVT::f32) {
5605 LHS = DAG.getNode(ISD::AND, dl, MVT::i32,
5606 bitcastf32Toi32(LHS, DAG), Mask);
5607 RHS = DAG.getNode(ISD::AND, dl, MVT::i32,
5608 bitcastf32Toi32(RHS, DAG), Mask);
5609 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5610 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
5611 Cmp);
5612 }
5613
5614 SDValue LHS1, LHS2;
5615 SDValue RHS1, RHS2;
5616 expandf64Toi32(LHS, DAG, LHS1, LHS2);
5617 expandf64Toi32(RHS, DAG, RHS1, RHS2);
5618 LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask);
5619 RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask);
5621 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5622 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5623 return DAG.getNode(ARMISD::BCC_i64, dl, MVT::Other, Ops);
5624 }
5625
5626 return SDValue();
5627}
5628
5629// Generate CMP + CMOV for integer abs.
5630SDValue ARMTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
5631 SDLoc DL(Op);
5632
5633 SDValue Neg = DAG.getNegative(Op.getOperand(0), DL, MVT::i32);
5634
5635 // Generate CMP & CMOV.
5636 SDValue Cmp = DAG.getNode(ARMISD::CMP, DL, FlagsVT, Op.getOperand(0),
5637 DAG.getConstant(0, DL, MVT::i32));
5638 return DAG.getNode(ARMISD::CMOV, DL, MVT::i32, Op.getOperand(0), Neg,
5639 DAG.getConstant(ARMCC::MI, DL, MVT::i32), Cmp);
5640}
5641
5643 ARMCC::CondCodes CondCode =
5644 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
5645 CondCode = ARMCC::getOppositeCondition(CondCode);
5646 return DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
5647}
5648
5649SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5650 SDValue Chain = Op.getOperand(0);
5651 SDValue Cond = Op.getOperand(1);
5652 SDValue Dest = Op.getOperand(2);
5653 SDLoc dl(Op);
5654
5655 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5656 // instruction.
5657 unsigned Opc = Cond.getOpcode();
5658 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5659 !Subtarget->isThumb1Only();
5660 if (Cond.getResNo() == 1 &&
5661 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5662 Opc == ISD::USUBO || OptimizeMul)) {
5663 // Only lower legal XALUO ops.
5664 if (!isTypeLegal(Cond->getValueType(0)))
5665 return SDValue();
5666
5667 // The actual operation with overflow check.
5668 SDValue Value, OverflowCmp;
5669 SDValue ARMcc;
5670 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
5671
5672 // Reverse the condition code.
5673 ARMcc = getInvertedARMCondCode(ARMcc, DAG);
5674
5675 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
5676 OverflowCmp);
5677 }
5678
5679 return SDValue();
5680}
5681
5682SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5683 SDValue Chain = Op.getOperand(0);
5684 ISD::CondCode CC = Op.getOperandCondCode(1);
5685 SDValue LHS = Op.getOperand(2);
5686 SDValue RHS = Op.getOperand(3);
5687 SDValue Dest = Op.getOperand(4);
5688 SDLoc dl(Op);
5689
5690 if (isUnsupportedFloatingType(LHS.getValueType())) {
5691 softenSetCCOperands(DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
5692
5693 // If softenSetCCOperands only returned one value, we should compare it to
5694 // zero.
5695 if (!RHS.getNode()) {
5696 RHS = DAG.getConstant(0, dl, LHS.getValueType());
5697 CC = ISD::SETNE;
5698 }
5699 }
5700
5701 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5702 // instruction.
5703 unsigned Opc = LHS.getOpcode();
5704 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5705 !Subtarget->isThumb1Only();
5706 if (LHS.getResNo() == 1 && (isOneConstant(RHS) || isNullConstant(RHS)) &&
5707 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5708 Opc == ISD::USUBO || OptimizeMul) &&
5709 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
5710 // Only lower legal XALUO ops.
5711 if (!isTypeLegal(LHS->getValueType(0)))
5712 return SDValue();
5713
5714 // The actual operation with overflow check.
5715 SDValue Value, OverflowCmp;
5716 SDValue ARMcc;
5717 std::tie(Value, OverflowCmp) = getARMXALUOOp(LHS.getValue(0), DAG, ARMcc);
5718
5719 if ((CC == ISD::SETNE) != isOneConstant(RHS)) {
5720 // Reverse the condition code.
5721 ARMcc = getInvertedARMCondCode(ARMcc, DAG);
5722 }
5723
5724 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
5725 OverflowCmp);
5726 }
5727
5728 if (LHS.getValueType() == MVT::i32) {
5729 SDValue ARMcc;
5730 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5731 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, Cmp);
5732 }
5733
5734 SDNodeFlags Flags = Op->getFlags();
5735 if (Flags.hasNoNaNs() &&
5736 DAG.getDenormalMode(MVT::f32) == DenormalMode::getIEEE() &&
5737 DAG.getDenormalMode(MVT::f64) == DenormalMode::getIEEE() &&
5738 (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETNE ||
5739 CC == ISD::SETUNE)) {
5740 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
5741 return Result;
5742 }
5743
5744 ARMCC::CondCodes CondCode, CondCode2;
5745 FPCCToARMCC(CC, CondCode, CondCode2);
5746
5747 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5748 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5749 SDValue Ops[] = {Chain, Dest, ARMcc, Cmp};
5750 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Ops);
5751 if (CondCode2 != ARMCC::AL) {
5752 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
5753 SDValue Ops[] = {Res, Dest, ARMcc, Cmp};
5754 Res = DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Ops);
5755 }
5756 return Res;
5757}
5758
5759SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5760 SDValue Chain = Op.getOperand(0);
5761 SDValue Table = Op.getOperand(1);
5762 SDValue Index = Op.getOperand(2);
5763 SDLoc dl(Op);
5764
5765 EVT PTy = getPointerTy(DAG.getDataLayout());
5766 JumpTableSDNode *JT = cast<JumpTableSDNode>(Table);
5767 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
5768 Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI);
5769 Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy));
5770 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index);
5771 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5772 // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table
5773 // which does another jump to the destination. This also makes it easier
5774 // to translate it to TBB / TBH later (Thumb2 only).
5775 // FIXME: This might not work if the function is extremely large.
5776 return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
5777 Addr, Op.getOperand(2), JTI);
5778 }
5779 if (isPositionIndependent() || Subtarget->isROPI()) {
5780 Addr =
5781 DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr,
5783 Chain = Addr.getValue(1);
5784 Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Addr);
5785 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5786 } else {
5787 Addr =
5788 DAG.getLoad(PTy, dl, Chain, Addr,
5790 Chain = Addr.getValue(1);
5791 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5792 }
5793}
5794
5796 EVT VT = Op.getValueType();
5797 SDLoc dl(Op);
5798
5799 if (Op.getValueType().getVectorElementType() == MVT::i32) {
5800 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5801 return Op;
5802 return DAG.UnrollVectorOp(Op.getNode());
5803 }
5804
5805 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5806
5807 EVT NewTy;
5808 const EVT OpTy = Op.getOperand(0).getValueType();
5809 if (OpTy == MVT::v4f32)
5810 NewTy = MVT::v4i32;
5811 else if (OpTy == MVT::v4f16 && HasFullFP16)
5812 NewTy = MVT::v4i16;
5813 else if (OpTy == MVT::v8f16 && HasFullFP16)
5814 NewTy = MVT::v8i16;
5815 else
5816 llvm_unreachable("Invalid type for custom lowering!");
5817
5818 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5819 return DAG.UnrollVectorOp(Op.getNode());
5820
5821 Op = DAG.getNode(Op.getOpcode(), dl, NewTy, Op.getOperand(0));
5822 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op);
5823}
5824
5825SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5826 EVT VT = Op.getValueType();
5827 if (VT.isVector())
5828 return LowerVectorFP_TO_INT(Op, DAG);
5829
5830 bool IsStrict = Op->isStrictFPOpcode();
5831 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
5832
5833 if (isUnsupportedFloatingType(SrcVal.getValueType())) {
5834 RTLIB::Libcall LC;
5835 if (Op.getOpcode() == ISD::FP_TO_SINT ||
5836 Op.getOpcode() == ISD::STRICT_FP_TO_SINT)
5837 LC = RTLIB::getFPTOSINT(SrcVal.getValueType(),
5838 Op.getValueType());
5839 else
5840 LC = RTLIB::getFPTOUINT(SrcVal.getValueType(),
5841 Op.getValueType());
5842 SDLoc Loc(Op);
5843 MakeLibCallOptions CallOptions;
5844 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
5846 std::tie(Result, Chain) = makeLibCall(DAG, LC, Op.getValueType(), SrcVal,
5847 CallOptions, Loc, Chain);
5848 return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
5849 }
5850
5851 return Op;
5852}
5853
5855 const ARMSubtarget *Subtarget) {
5856 EVT VT = Op.getValueType();
5857 EVT ToVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
5858 EVT FromVT = Op.getOperand(0).getValueType();
5859
5860 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5861 return Op;
5862 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5863 Subtarget->hasFP64())
5864 return Op;
5865 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5866 Subtarget->hasFullFP16())
5867 return Op;
5868 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5869 Subtarget->hasMVEFloatOps())
5870 return Op;
5871 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5872 Subtarget->hasMVEFloatOps())
5873 return Op;
5874
5875 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5876 return SDValue();
5877
5878 SDLoc DL(Op);
5879 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
5880 unsigned BW = ToVT.getScalarSizeInBits() - IsSigned;
5881 SDValue CVT = DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
5882 DAG.getValueType(VT.getScalarType()));
5883 SDValue Max = DAG.getNode(IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT, CVT,
5884 DAG.getConstant((1 << BW) - 1, DL, VT));
5885 if (IsSigned)
5886 Max = DAG.getNode(ISD::SMAX, DL, VT, Max,
5887 DAG.getSignedConstant(-(1 << BW), DL, VT));
5888 return Max;
5889}
5890
5892 EVT VT = Op.getValueType();
5893 SDLoc dl(Op);
5894
5895 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5896 if (VT.getVectorElementType() == MVT::f32)
5897 return Op;
5898 return DAG.UnrollVectorOp(Op.getNode());
5899 }
5900
5901 assert((Op.getOperand(0).getValueType() == MVT::v4i16 ||
5902 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5903 "Invalid type for custom lowering!");
5904
5905 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5906
5907 EVT DestVecType;
5908 if (VT == MVT::v4f32)
5909 DestVecType = MVT::v4i32;
5910 else if (VT == MVT::v4f16 && HasFullFP16)
5911 DestVecType = MVT::v4i16;
5912 else if (VT == MVT::v8f16 && HasFullFP16)
5913 DestVecType = MVT::v8i16;
5914 else
5915 return DAG.UnrollVectorOp(Op.getNode());
5916
5917 unsigned CastOpc;
5918 unsigned Opc;
5919 switch (Op.getOpcode()) {
5920 default: llvm_unreachable("Invalid opcode!");
5921 case ISD::SINT_TO_FP:
5922 CastOpc = ISD::SIGN_EXTEND;
5924 break;
5925 case ISD::UINT_TO_FP:
5926 CastOpc = ISD::ZERO_EXTEND;
5928 break;
5929 }
5930
5931 Op = DAG.getNode(CastOpc, dl, DestVecType, Op.getOperand(0));
5932 return DAG.getNode(Opc, dl, VT, Op);
5933}
5934
5935SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
5936 EVT VT = Op.getValueType();
5937 if (VT.isVector())
5938 return LowerVectorINT_TO_FP(Op, DAG);
5939
5940 bool IsStrict = Op->isStrictFPOpcode();
5941 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
5942
5943 if (isUnsupportedFloatingType(VT)) {
5944 RTLIB::Libcall LC;
5945 if (Op.getOpcode() == ISD::SINT_TO_FP ||
5946 Op.getOpcode() == ISD::STRICT_SINT_TO_FP)
5947 LC = RTLIB::getSINTTOFP(SrcVal.getValueType(), Op.getValueType());
5948 else
5949 LC = RTLIB::getUINTTOFP(SrcVal.getValueType(), Op.getValueType());
5950 SDLoc Loc(Op);
5951 MakeLibCallOptions CallOptions;
5952 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
5954 std::tie(Result, Chain) = makeLibCall(DAG, LC, Op.getValueType(), SrcVal,
5955 CallOptions, Loc, Chain);
5956 return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
5957 }
5958
5959 return Op;
5960}
5961
5962SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
5963 // Implement fcopysign with a fabs and a conditional fneg.
5964 SDValue Tmp0 = Op.getOperand(0);
5965 SDValue Tmp1 = Op.getOperand(1);
5966 SDLoc dl(Op);
5967 EVT VT = Op.getValueType();
5968 EVT SrcVT = Tmp1.getValueType();
5969 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST ||
5970 Tmp0.getOpcode() == ARMISD::VMOVDRR;
5971 bool UseNEON = !InGPR && Subtarget->hasNEON();
5972
5973 if (UseNEON) {
5974 // Use VBSL to copy the sign bit.
5975 unsigned EncodedVal = ARM_AM::createVMOVModImm(0x6, 0x80);
5976 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
5977 DAG.getTargetConstant(EncodedVal, dl, MVT::i32));
5978 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
5979 if (VT == MVT::f64)
5980 Mask = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
5981 DAG.getNode(ISD::BITCAST, dl, OpVT, Mask),
5982 DAG.getConstant(32, dl, MVT::i32));
5983 else /*if (VT == MVT::f32)*/
5984 Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0);
5985 if (SrcVT == MVT::f32) {
5986 Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1);
5987 if (VT == MVT::f64)
5988 Tmp1 = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
5989 DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1),
5990 DAG.getConstant(32, dl, MVT::i32));
5991 } else if (VT == MVT::f32)
5992 Tmp1 = DAG.getNode(ARMISD::VSHRuIMM, dl, MVT::v1i64,
5993 DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1),
5994 DAG.getConstant(32, dl, MVT::i32));
5995 Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0);
5996 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1);
5997
5999 dl, MVT::i32);
6000 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes);
6001 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
6002 DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes));
6003
6004 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT,
6005 DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask),
6006 DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot));
6007 if (VT == MVT::f32) {
6008 Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res);
6009 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
6010 DAG.getConstant(0, dl, MVT::i32));
6011 } else {
6012 Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res);
6013 }
6014
6015 return Res;
6016 }
6017
6018 // Bitcast operand 1 to i32.
6019 if (SrcVT == MVT::f64)
6020 Tmp1 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
6021 Tmp1).getValue(1);
6022 Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1);
6023
6024 // Or in the signbit with integer operations.
6025 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
6026 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
6027 Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1);
6028 if (VT == MVT::f32) {
6029 Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32,
6030 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2);
6031 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
6032 DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1));
6033 }
6034
6035 // f64: Or the high part with signbit and then combine two parts.
6036 Tmp0 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
6037 Tmp0);
6038 SDValue Lo = Tmp0.getValue(0);
6039 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2);
6040 Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1);
6041 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
6042}
6043
6044SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
6046 MachineFrameInfo &MFI = MF.getFrameInfo();
6047 MFI.setReturnAddressIsTaken(true);
6048
6049 EVT VT = Op.getValueType();
6050 SDLoc dl(Op);
6051 unsigned Depth = Op.getConstantOperandVal(0);
6052 if (Depth) {
6053 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
6054 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
6055 return DAG.getLoad(VT, dl, DAG.getEntryNode(),
6056 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
6057 MachinePointerInfo());
6058 }
6059
6060 // Return LR, which contains the return address. Mark it an implicit live-in.
6061 Register Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
6062 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
6063}
6064
6065SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
6066 const ARMBaseRegisterInfo &ARI =
6067 *static_cast<const ARMBaseRegisterInfo*>(RegInfo);
6069 MachineFrameInfo &MFI = MF.getFrameInfo();
6070 MFI.setFrameAddressIsTaken(true);
6071
6072 EVT VT = Op.getValueType();
6073 SDLoc dl(Op); // FIXME probably not meaningful
6074 unsigned Depth = Op.getConstantOperandVal(0);
6075 Register FrameReg = ARI.getFrameRegister(MF);
6076 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
6077 while (Depth--)
6078 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
6079 MachinePointerInfo());
6080 return FrameAddr;
6081}
6082
6083// FIXME? Maybe this could be a TableGen attribute on some registers and
6084// this table could be generated automatically from RegInfo.
6085Register ARMTargetLowering::getRegisterByName(const char* RegName, LLT VT,
6086 const MachineFunction &MF) const {
6087 return StringSwitch<Register>(RegName)
6088 .Case("sp", ARM::SP)
6089 .Default(Register());
6090}
6091
6092// Result is 64 bit value so split into two 32 bit values and return as a
6093// pair of values.
6095 SelectionDAG &DAG) {
6096 SDLoc DL(N);
6097
6098 // This function is only supposed to be called for i64 type destination.
6099 assert(N->getValueType(0) == MVT::i64
6100 && "ExpandREAD_REGISTER called for non-i64 type result.");
6101
6103 DAG.getVTList(MVT::i32, MVT::i32, MVT::Other),
6104 N->getOperand(0),
6105 N->getOperand(1));
6106
6107 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0),
6108 Read.getValue(1)));
6109 Results.push_back(Read.getValue(2)); // Chain
6110}
6111
6112/// \p BC is a bitcast that is about to be turned into a VMOVDRR.
6113/// When \p DstVT, the destination type of \p BC, is on the vector
6114/// register bank and the source of bitcast, \p Op, operates on the same bank,
6115/// it might be possible to combine them, such that everything stays on the
6116/// vector register bank.
6117/// \p return The node that would replace \p BT, if the combine
6118/// is possible.
6120 SelectionDAG &DAG) {
6121 SDValue Op = BC->getOperand(0);
6122 EVT DstVT = BC->getValueType(0);
6123
6124 // The only vector instruction that can produce a scalar (remember,
6125 // since the bitcast was about to be turned into VMOVDRR, the source
6126 // type is i64) from a vector is EXTRACT_VECTOR_ELT.
6127 // Moreover, we can do this combine only if there is one use.
6128 // Finally, if the destination type is not a vector, there is not
6129 // much point on forcing everything on the vector bank.
6130 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6131 !Op.hasOneUse())
6132 return SDValue();
6133
6134 // If the index is not constant, we will introduce an additional
6135 // multiply that will stick.
6136 // Give up in that case.
6137 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Op.getOperand(1));
6138 if (!Index)
6139 return SDValue();
6140 unsigned DstNumElt = DstVT.getVectorNumElements();
6141
6142 // Compute the new index.
6143 const APInt &APIntIndex = Index->getAPIntValue();
6144 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt);
6145 NewIndex *= APIntIndex;
6146 // Check if the new constant index fits into i32.
6147 if (NewIndex.getBitWidth() > 32)
6148 return SDValue();
6149
6150 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) ->
6151 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M)
6152 SDLoc dl(Op);
6153 SDValue ExtractSrc = Op.getOperand(0);
6154 EVT VecVT = EVT::getVectorVT(
6155 *DAG.getContext(), DstVT.getScalarType(),
6156 ExtractSrc.getValueType().getVectorNumElements() * DstNumElt);
6157 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
6158 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast,
6159 DAG.getConstant(NewIndex.getZExtValue(), dl, MVT::i32));
6160}
6161
6162/// ExpandBITCAST - If the target supports VFP, this function is called to
6163/// expand a bit convert where either the source or destination type is i64 to
6164/// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
6165/// operand type is illegal (e.g., v2f32 for a target that doesn't support
6166/// vectors), since the legalizer won't know what to do with that.
6167SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
6168 const ARMSubtarget *Subtarget) const {
6169 SDLoc dl(N);
6170 SDValue Op = N->getOperand(0);
6171
6172 // This function is only supposed to be called for i16 and i64 types, either
6173 // as the source or destination of the bit convert.
6174 EVT SrcVT = Op.getValueType();
6175 EVT DstVT = N->getValueType(0);
6176
6177 if ((SrcVT == MVT::f16 || SrcVT == MVT::bf16) &&
6178 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6179 return SDValue(N, 0);
6180
6181 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6182 (DstVT == MVT::f16 || DstVT == MVT::bf16)) {
6183 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), MVT::i32, Op);
6184 if (Subtarget->hasFullFP16())
6185 return DAG.getNode(ARMISD::VMOVhr, dl, DstVT, Ext);
6186 return DAG.getNode(ARMISD::VMOVsr, dl, DstVT, Ext);
6187 }
6188
6189 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6190 (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
6191 if (Subtarget->hasFullFP16() && !Subtarget->hasBF16())
6192 Op = DAG.getBitcast(MVT::f16, Op);
6193 return DAG.getNode(
6194 ISD::TRUNCATE, SDLoc(N), DstVT,
6195 MoveFromHPR(SDLoc(N), DAG, MVT::i32, SrcVT.getSimpleVT(), Op));
6196 }
6197
6198 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6199 return SDValue();
6200
6201 // Turn i64->f64 into VMOVDRR.
6202 if (SrcVT == MVT::i64 && isTypeLegal(DstVT)) {
6203 // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
6204 // if we can combine the bitcast with its source.
6206 return Val;
6207 SDValue Lo, Hi;
6208 std::tie(Lo, Hi) = DAG.SplitScalar(Op, dl, MVT::i32, MVT::i32);
6209 return DAG.getNode(ISD::BITCAST, dl, DstVT,
6210 DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi));
6211 }
6212
6213 // Turn f64->i64 into VMOVRRD.
6214 if (DstVT == MVT::i64 && isTypeLegal(SrcVT)) {
6215 SDValue Cvt;
6216 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
6217 SrcVT.getVectorNumElements() > 1)
6218 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
6219 DAG.getVTList(MVT::i32, MVT::i32),
6220 DAG.getNode(ARMISD::VREV64, dl, SrcVT, Op));
6221 else
6222 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
6223 DAG.getVTList(MVT::i32, MVT::i32), Op);
6224 // Merge the pieces into a single i64 value.
6225 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1));
6226 }
6227
6228 return SDValue();
6229}
6230
6231/// getZeroVector - Returns a vector of specified type with all zero elements.
6232/// Zero vectors are used to represent vector negation and in those cases
6233/// will be implemented with the NEON VNEG instruction. However, VNEG does
6234/// not support i64 elements, so sometimes the zero vectors will need to be
6235/// explicitly constructed. Regardless, use a canonical VMOV to create the
6236/// zero vector.
6237static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
6238 assert(VT.isVector() && "Expected a vector type");
6239 // The canonical modified immediate encoding of a zero vector is....0!
6240 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
6241 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
6242 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
6243 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
6244}
6245
6246/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
6247/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6248SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
6249 SelectionDAG &DAG) const {
6250 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6251 EVT VT = Op.getValueType();
6252 unsigned VTBits = VT.getSizeInBits();
6253 SDLoc dl(Op);
6254 SDValue ShOpLo = Op.getOperand(0);
6255 SDValue ShOpHi = Op.getOperand(1);
6256 SDValue ShAmt = Op.getOperand(2);
6257 SDValue ARMcc;
6258 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
6259
6260 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
6261
6262 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6263 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
6264 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
6265 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6266 DAG.getConstant(VTBits, dl, MVT::i32));
6267 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
6268 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6269 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6270 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6271 ISD::SETGE, ARMcc, DAG, dl);
6272 SDValue Lo =
6273 DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift, ARMcc, CmpLo);
6274
6275 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
6276 SDValue HiBigShift = Opc == ISD::SRA
6277 ? DAG.getNode(Opc, dl, VT, ShOpHi,
6278 DAG.getConstant(VTBits - 1, dl, VT))
6279 : DAG.getConstant(0, dl, VT);
6280 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6281 ISD::SETGE, ARMcc, DAG, dl);
6282 SDValue Hi =
6283 DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
6284
6285 SDValue Ops[2] = { Lo, Hi };
6286 return DAG.getMergeValues(Ops, dl);
6287}
6288
6289/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
6290/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6291SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
6292 SelectionDAG &DAG) const {
6293 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6294 EVT VT = Op.getValueType();
6295 unsigned VTBits = VT.getSizeInBits();
6296 SDLoc dl(Op);
6297 SDValue ShOpLo = Op.getOperand(0);
6298 SDValue ShOpHi = Op.getOperand(1);
6299 SDValue ShAmt = Op.getOperand(2);
6300 SDValue ARMcc;
6301
6302 assert(Op.getOpcode() == ISD::SHL_PARTS);
6303 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6304 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
6305 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
6306 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
6307 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6308
6309 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6310 DAG.getConstant(VTBits, dl, MVT::i32));
6311 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
6312 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6313 ISD::SETGE, ARMcc, DAG, dl);
6314 SDValue Hi =
6315 DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
6316
6317 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6318 ISD::SETGE, ARMcc, DAG, dl);
6319 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
6320 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift,
6321 DAG.getConstant(0, dl, VT), ARMcc, CmpLo);
6322
6323 SDValue Ops[2] = { Lo, Hi };
6324 return DAG.getMergeValues(Ops, dl);
6325}
6326
6327SDValue ARMTargetLowering::LowerGET_ROUNDING(SDValue Op,
6328 SelectionDAG &DAG) const {
6329 // The rounding mode is in bits 23:22 of the FPSCR.
6330 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
6331 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
6332 // so that the shift + and get folded into a bitfield extract.
6333 SDLoc dl(Op);
6334 SDValue Chain = Op.getOperand(0);
6335 SDValue Ops[] = {Chain,
6336 DAG.getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6337
6338 SDValue FPSCR =
6339 DAG.getNode(ISD::INTRINSIC_W_CHAIN, dl, {MVT::i32, MVT::Other}, Ops);
6340 Chain = FPSCR.getValue(1);
6341 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
6342 DAG.getConstant(1U << 22, dl, MVT::i32));
6343 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
6344 DAG.getConstant(22, dl, MVT::i32));
6345 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
6346 DAG.getConstant(3, dl, MVT::i32));
6347 return DAG.getMergeValues({And, Chain}, dl);
6348}
6349
6350SDValue ARMTargetLowering::LowerSET_ROUNDING(SDValue Op,
6351 SelectionDAG &DAG) const {
6352 SDLoc DL(Op);
6353 SDValue Chain = Op->getOperand(0);
6354 SDValue RMValue = Op->getOperand(1);
6355
6356 // The rounding mode is in bits 23:22 of the FPSCR.
6357 // The llvm.set.rounding argument value to ARM rounding mode value mapping
6358 // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
6359 // ((arg - 1) & 3) << 22).
6360 //
6361 // It is expected that the argument of llvm.set.rounding is within the
6362 // segment [0, 3], so NearestTiesToAway (4) is not handled here. It is
6363 // responsibility of the code generated llvm.set.rounding to ensure this
6364 // condition.
6365
6366 // Calculate new value of FPSCR[23:22].
6367 RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue,
6368 DAG.getConstant(1, DL, MVT::i32));
6369 RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue,
6370 DAG.getConstant(0x3, DL, MVT::i32));
6371 RMValue = DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue,
6372 DAG.getConstant(ARM::RoundingBitsPos, DL, MVT::i32));
6373
6374 // Get current value of FPSCR.
6375 SDValue Ops[] = {Chain,
6376 DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
6377 SDValue FPSCR =
6378 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
6379 Chain = FPSCR.getValue(1);
6380 FPSCR = FPSCR.getValue(0);
6381
6382 // Put new rounding mode into FPSCR[23:22].
6383 const unsigned RMMask = ~(ARM::Rounding::rmMask << ARM::RoundingBitsPos);
6384 FPSCR = DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR,
6385 DAG.getConstant(RMMask, DL, MVT::i32));
6386 FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCR, RMValue);
6387 SDValue Ops2[] = {
6388 Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR};
6389 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
6390}
6391
6392SDValue ARMTargetLowering::LowerSET_FPMODE(SDValue Op,
6393 SelectionDAG &DAG) const {
6394 SDLoc DL(Op);
6395 SDValue Chain = Op->getOperand(0);
6396 SDValue Mode = Op->getOperand(1);
6397
6398 // Generate nodes to build:
6399 // FPSCR = (FPSCR & FPStatusBits) | (Mode & ~FPStatusBits)
6400 SDValue Ops[] = {Chain,
6401 DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
6402 SDValue FPSCR =
6403 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
6404 Chain = FPSCR.getValue(1);
6405 FPSCR = FPSCR.getValue(0);
6406
6407 SDValue FPSCRMasked =
6408 DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR,
6409 DAG.getConstant(ARM::FPStatusBits, DL, MVT::i32));
6410 SDValue InputMasked =
6411 DAG.getNode(ISD::AND, DL, MVT::i32, Mode,
6412 DAG.getConstant(~ARM::FPStatusBits, DL, MVT::i32));
6413 FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCRMasked, InputMasked);
6414
6415 SDValue Ops2[] = {
6416 Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR};
6417 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
6418}
6419
6420SDValue ARMTargetLowering::LowerRESET_FPMODE(SDValue Op,
6421 SelectionDAG &DAG) const {
6422 SDLoc DL(Op);
6423 SDValue Chain = Op->getOperand(0);
6424
6425 // To get the default FP mode all control bits are cleared:
6426 // FPSCR = FPSCR & (FPStatusBits | FPReservedBits)
6427 SDValue Ops[] = {Chain,
6428 DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
6429 SDValue FPSCR =
6430 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
6431 Chain = FPSCR.getValue(1);
6432 FPSCR = FPSCR.getValue(0);
6433
6434 SDValue FPSCRMasked = DAG.getNode(
6435 ISD::AND, DL, MVT::i32, FPSCR,
6437 SDValue Ops2[] = {Chain,
6438 DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32),
6439 FPSCRMasked};
6440 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
6441}
6442
6444 const ARMSubtarget *ST) {
6445 SDLoc dl(N);
6446 EVT VT = N->getValueType(0);
6447 if (VT.isVector() && ST->hasNEON()) {
6448
6449 // Compute the least significant set bit: LSB = X & -X
6450 SDValue X = N->getOperand(0);
6451 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
6452 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
6453
6455
6456 if (ElemTy == MVT::i8) {
6457 // Compute with: cttz(x) = ctpop(lsb - 1)
6458 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6459 DAG.getTargetConstant(1, dl, ElemTy));
6460 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
6461 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
6462 }
6463
6464 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6465 (N->getOpcode() == ISD::CTTZ_ZERO_POISON)) {
6466 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
6467 unsigned NumBits = ElemTy.getSizeInBits();
6468 SDValue WidthMinus1 =
6469 DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6470 DAG.getTargetConstant(NumBits - 1, dl, ElemTy));
6471 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
6472 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ);
6473 }
6474
6475 // Compute with: cttz(x) = ctpop(lsb - 1)
6476
6477 // Compute LSB - 1.
6478 SDValue Bits;
6479 if (ElemTy == MVT::i64) {
6480 // Load constant 0xffff'ffff'ffff'ffff to register.
6481 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6482 DAG.getTargetConstant(0x1eff, dl, MVT::i32));
6483 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF);
6484 } else {
6485 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6486 DAG.getTargetConstant(1, dl, ElemTy));
6487 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
6488 }
6489 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
6490 }
6491
6492 if (!ST->hasV6T2Ops())
6493 return SDValue();
6494
6495 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0));
6496 return DAG.getNode(ISD::CTLZ, dl, VT, rbit);
6497}
6498
6500 const ARMSubtarget *ST) {
6501 EVT VT = N->getValueType(0);
6502 SDLoc DL(N);
6503
6504 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.");
6505 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6506 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6507 "Unexpected type for custom ctpop lowering");
6508
6510 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
6511 SDValue Res = DAG.getBitcast(VT8Bit, N->getOperand(0));
6512 Res = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Res);
6513
6514 // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
6515 unsigned EltSize = 8;
6516 unsigned NumElts = VT.is64BitVector() ? 8 : 16;
6517 while (EltSize != VT.getScalarSizeInBits()) {
6519 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddlu, DL,
6520 TLI.getPointerTy(DAG.getDataLayout())));
6521 Ops.push_back(Res);
6522
6523 EltSize *= 2;
6524 NumElts /= 2;
6525 MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
6526 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WidenVT, Ops);
6527 }
6528
6529 return Res;
6530}
6531
6532/// Getvshiftimm - Check if this is a valid build_vector for the immediate
6533/// operand of a vector shift operation, where all the elements of the
6534/// build_vector must have the same constant integer value.
6535static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
6536 // Ignore bit_converts.
6537 while (Op.getOpcode() == ISD::BITCAST)
6538 Op = Op.getOperand(0);
6540 APInt SplatBits, SplatUndef;
6541 unsigned SplatBitSize;
6542 bool HasAnyUndefs;
6543 if (!BVN ||
6544 !BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6545 ElementBits) ||
6546 SplatBitSize > ElementBits)
6547 return false;
6548 Cnt = SplatBits.getSExtValue();
6549 return true;
6550}
6551
6552/// isVShiftLImm - Check if this is a valid build_vector for the immediate
6553/// operand of a vector shift left operation. That value must be in the range:
6554/// 0 <= Value < ElementBits for a left shift; or
6555/// 0 <= Value <= ElementBits for a long left shift.
6556static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
6557 assert(VT.isVector() && "vector shift count is not a vector type");
6558 int64_t ElementBits = VT.getScalarSizeInBits();
6559 if (!getVShiftImm(Op, ElementBits, Cnt))
6560 return false;
6561 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6562}
6563
6564/// isVShiftRImm - Check if this is a valid build_vector for the immediate
6565/// operand of a vector shift right operation. For a shift opcode, the value
6566/// is positive, but for an intrinsic the value count must be negative. The
6567/// absolute value must be in the range:
6568/// 1 <= |Value| <= ElementBits for a right shift; or
6569/// 1 <= |Value| <= ElementBits/2 for a narrow right shift.
6570static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
6571 int64_t &Cnt) {
6572 assert(VT.isVector() && "vector shift count is not a vector type");
6573 int64_t ElementBits = VT.getScalarSizeInBits();
6574 if (!getVShiftImm(Op, ElementBits, Cnt))
6575 return false;
6576 if (!isIntrinsic)
6577 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6578 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6579 Cnt = -Cnt;
6580 return true;
6581 }
6582 return false;
6583}
6584
6586 const ARMSubtarget *ST) {
6587 EVT VT = N->getValueType(0);
6588 SDLoc dl(N);
6589 int64_t Cnt;
6590
6591 if (!VT.isVector())
6592 return SDValue();
6593
6594 // We essentially have two forms here. Shift by an immediate and shift by a
6595 // vector register (there are also shift by a gpr, but that is just handled
6596 // with a tablegen pattern). We cannot easily match shift by an immediate in
6597 // tablegen so we do that here and generate a VSHLIMM/VSHRsIMM/VSHRuIMM.
6598 // For shifting by a vector, we don't have VSHR, only VSHL (which can be
6599 // signed or unsigned, and a negative shift indicates a shift right).
6600 if (N->getOpcode() == ISD::SHL) {
6601 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt))
6602 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0),
6603 DAG.getConstant(Cnt, dl, MVT::i32));
6604 return DAG.getNode(ARMISD::VSHLu, dl, VT, N->getOperand(0),
6605 N->getOperand(1));
6606 }
6607
6608 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
6609 "unexpected vector shift opcode");
6610
6611 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
6612 unsigned VShiftOpc =
6613 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
6614 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
6615 DAG.getConstant(Cnt, dl, MVT::i32));
6616 }
6617
6618 // Other right shifts we don't have operations for (we use a shift left by a
6619 // negative number).
6620 EVT ShiftVT = N->getOperand(1).getValueType();
6621 SDValue NegatedCount = DAG.getNode(
6622 ISD::SUB, dl, ShiftVT, getZeroVector(ShiftVT, DAG, dl), N->getOperand(1));
6623 unsigned VShiftOpc =
6624 (N->getOpcode() == ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu);
6625 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), NegatedCount);
6626}
6627
6629 const ARMSubtarget *ST) {
6630 EVT VT = N->getValueType(0);
6631 SDLoc dl(N);
6632
6633 // We can get here for a node like i32 = ISD::SHL i32, i64
6634 if (VT != MVT::i64)
6635 return SDValue();
6636
6637 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA ||
6638 N->getOpcode() == ISD::SHL) &&
6639 "Unknown shift to lower!");
6640
6641 unsigned ShOpc = N->getOpcode();
6642 if (ST->hasMVEIntegerOps()) {
6643 SDValue ShAmt = N->getOperand(1);
6644 unsigned ShPartsOpc = ARMISD::LSLL;
6646
6647 // If the shift amount is greater than 32 or has a greater bitwidth than 64
6648 // then do the default optimisation
6649 if ((!Con && ShAmt->getValueType(0).getSizeInBits() > 64) ||
6650 (Con && (Con->getAPIntValue() == 0 || Con->getAPIntValue().uge(32))))
6651 return SDValue();
6652
6653 // Extract the lower 32 bits of the shift amount if it's not an i32
6654 if (ShAmt->getValueType(0) != MVT::i32)
6655 ShAmt = DAG.getZExtOrTrunc(ShAmt, dl, MVT::i32);
6656
6657 if (ShOpc == ISD::SRL) {
6658 if (!Con)
6659 // There is no t2LSRLr instruction so negate and perform an lsll if the
6660 // shift amount is in a register, emulating a right shift.
6661 ShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6662 DAG.getConstant(0, dl, MVT::i32), ShAmt);
6663 else
6664 // Else generate an lsrl on the immediate shift amount
6665 ShPartsOpc = ARMISD::LSRL;
6666 } else if (ShOpc == ISD::SRA)
6667 ShPartsOpc = ARMISD::ASRL;
6668
6669 // Split Lower/Upper 32 bits of the destination/source
6670 SDValue Lo, Hi;
6671 std::tie(Lo, Hi) =
6672 DAG.SplitScalar(N->getOperand(0), dl, MVT::i32, MVT::i32);
6673 // Generate the shift operation as computed above
6674 Lo = DAG.getNode(ShPartsOpc, dl, DAG.getVTList(MVT::i32, MVT::i32), Lo, Hi,
6675 ShAmt);
6676 // The upper 32 bits come from the second return value of lsll
6677 Hi = SDValue(Lo.getNode(), 1);
6678 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
6679 }
6680
6681 // We only lower SRA, SRL of 1 here, all others use generic lowering.
6682 if (!isOneConstant(N->getOperand(1)) || N->getOpcode() == ISD::SHL)
6683 return SDValue();
6684
6685 // If we are in thumb mode, we don't have RRX.
6686 if (ST->isThumb1Only())
6687 return SDValue();
6688
6689 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr.
6690 SDValue Lo, Hi;
6691 std::tie(Lo, Hi) = DAG.SplitScalar(N->getOperand(0), dl, MVT::i32, MVT::i32);
6692
6693 // First, build a LSRS1/ASRS1 op, which shifts the top part by one and
6694 // captures the shifted out bit into a carry flag.
6695 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::LSRS1 : ARMISD::ASRS1;
6696 Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, FlagsVT), Hi);
6697
6698 // The low part is an ARMISD::RRX operand, which shifts the carry in.
6699 Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1));
6700
6701 // Merge the pieces into a single i64 value.
6702 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
6703}
6704
6706 const ARMSubtarget *ST) {
6707 bool Invert = false;
6708 bool Swap = false;
6709 unsigned Opc = ARMCC::AL;
6710
6711 SDValue Op0 = Op.getOperand(0);
6712 SDValue Op1 = Op.getOperand(1);
6713 SDValue CC = Op.getOperand(2);
6714 EVT VT = Op.getValueType();
6715 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
6716 SDLoc dl(Op);
6717
6718 EVT CmpVT;
6719 if (ST->hasNEON())
6721 else {
6722 assert(ST->hasMVEIntegerOps() &&
6723 "No hardware support for integer vector comparison!");
6724
6725 if (Op.getValueType().getVectorElementType() != MVT::i1)
6726 return SDValue();
6727
6728 // Make sure we expand floating point setcc to scalar if we do not have
6729 // mve.fp, so that we can handle them from there.
6730 if (Op0.getValueType().isFloatingPoint() && !ST->hasMVEFloatOps())
6731 return SDValue();
6732
6733 CmpVT = VT;
6734 }
6735
6736 if (Op0.getValueType().getVectorElementType() == MVT::i64 &&
6737 (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) {
6738 // Special-case integer 64-bit equality comparisons. They aren't legal,
6739 // but they can be lowered with a few vector instructions.
6740 unsigned CmpElements = CmpVT.getVectorNumElements() * 2;
6741 EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, CmpElements);
6742 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0);
6743 SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1);
6744 SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1,
6745 DAG.getCondCode(ISD::SETEQ));
6746 SDValue Reversed = DAG.getNode(ARMISD::VREV64, dl, SplitVT, Cmp);
6747 SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed);
6748 Merged = DAG.getNode(ISD::BITCAST, dl, CmpVT, Merged);
6749 if (SetCCOpcode == ISD::SETNE)
6750 Merged = DAG.getNOT(dl, Merged, CmpVT);
6751 Merged = DAG.getSExtOrTrunc(Merged, dl, VT);
6752 return Merged;
6753 }
6754
6755 if (CmpVT.getVectorElementType() == MVT::i64)
6756 // 64-bit comparisons are not legal in general.
6757 return SDValue();
6758
6759 if (Op1.getValueType().isFloatingPoint()) {
6760 switch (SetCCOpcode) {
6761 default: llvm_unreachable("Illegal FP comparison");
6762 case ISD::SETUNE:
6763 case ISD::SETNE:
6764 if (ST->hasMVEFloatOps()) {
6765 Opc = ARMCC::NE; break;
6766 } else {
6767 Invert = true; [[fallthrough]];
6768 }
6769 case ISD::SETOEQ:
6770 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6771 case ISD::SETOLT:
6772 case ISD::SETLT: Swap = true; [[fallthrough]];
6773 case ISD::SETOGT:
6774 case ISD::SETGT: Opc = ARMCC::GT; break;
6775 case ISD::SETOLE:
6776 case ISD::SETLE: Swap = true; [[fallthrough]];
6777 case ISD::SETOGE:
6778 case ISD::SETGE: Opc = ARMCC::GE; break;
6779 case ISD::SETUGE: Swap = true; [[fallthrough]];
6780 case ISD::SETULE: Invert = true; Opc = ARMCC::GT; break;
6781 case ISD::SETUGT: Swap = true; [[fallthrough]];
6782 case ISD::SETULT: Invert = true; Opc = ARMCC::GE; break;
6783 case ISD::SETUEQ: Invert = true; [[fallthrough]];
6784 case ISD::SETONE: {
6785 // Expand this to (OLT | OGT).
6786 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6787 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6788 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6789 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6790 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6791 if (Invert)
6792 Result = DAG.getNOT(dl, Result, VT);
6793 return Result;
6794 }
6795 case ISD::SETUO: Invert = true; [[fallthrough]];
6796 case ISD::SETO: {
6797 // Expand this to (OLT | OGE).
6798 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6799 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6800 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6801 DAG.getConstant(ARMCC::GE, dl, MVT::i32));
6802 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6803 if (Invert)
6804 Result = DAG.getNOT(dl, Result, VT);
6805 return Result;
6806 }
6807 }
6808 } else {
6809 // Integer comparisons.
6810 switch (SetCCOpcode) {
6811 default: llvm_unreachable("Illegal integer comparison");
6812 case ISD::SETNE:
6813 if (ST->hasMVEIntegerOps()) {
6814 Opc = ARMCC::NE; break;
6815 } else {
6816 Invert = true; [[fallthrough]];
6817 }
6818 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6819 case ISD::SETLT: Swap = true; [[fallthrough]];
6820 case ISD::SETGT: Opc = ARMCC::GT; break;
6821 case ISD::SETLE: Swap = true; [[fallthrough]];
6822 case ISD::SETGE: Opc = ARMCC::GE; break;
6823 case ISD::SETULT: Swap = true; [[fallthrough]];
6824 case ISD::SETUGT: Opc = ARMCC::HI; break;
6825 case ISD::SETULE: Swap = true; [[fallthrough]];
6826 case ISD::SETUGE: Opc = ARMCC::HS; break;
6827 }
6828
6829 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
6830 if (ST->hasNEON() && Opc == ARMCC::EQ) {
6831 SDValue AndOp;
6833 AndOp = Op0;
6834 else if (ISD::isBuildVectorAllZeros(Op0))
6835 AndOp = Op1;
6836
6837 // Ignore bitconvert.
6838 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
6839 AndOp = AndOp.getOperand(0);
6840
6841 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
6842 Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0));
6843 Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1));
6844 SDValue Result = DAG.getNode(ARMISD::VTST, dl, CmpVT, Op0, Op1);
6845 if (!Invert)
6846 Result = DAG.getNOT(dl, Result, VT);
6847 return Result;
6848 }
6849 }
6850 }
6851
6852 if (Swap)
6853 std::swap(Op0, Op1);
6854
6855 // If one of the operands is a constant vector zero, attempt to fold the
6856 // comparison to a specialized compare-against-zero form.
6857 if (ISD::isBuildVectorAllZeros(Op0) &&
6858 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::EQ ||
6859 Opc == ARMCC::NE)) {
6860 if (Opc == ARMCC::GE)
6861 Opc = ARMCC::LE;
6862 else if (Opc == ARMCC::GT)
6863 Opc = ARMCC::LT;
6864 std::swap(Op0, Op1);
6865 }
6866
6867 SDValue Result;
6868 if (ISD::isBuildVectorAllZeros(Op1) &&
6869 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::LE ||
6870 Opc == ARMCC::LT || Opc == ARMCC::NE || Opc == ARMCC::EQ))
6871 Result = DAG.getNode(ARMISD::VCMPZ, dl, CmpVT, Op0,
6872 DAG.getConstant(Opc, dl, MVT::i32));
6873 else
6874 Result = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6875 DAG.getConstant(Opc, dl, MVT::i32));
6876
6877 Result = DAG.getSExtOrTrunc(Result, dl, VT);
6878
6879 if (Invert)
6880 Result = DAG.getNOT(dl, Result, VT);
6881
6882 return Result;
6883}
6884
6886 SDValue LHS = Op.getOperand(0);
6887 SDValue RHS = Op.getOperand(1);
6888
6889 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
6890
6891 SDValue Carry = Op.getOperand(2);
6892 SDValue Cond = Op.getOperand(3);
6893 SDLoc DL(Op);
6894
6895 // ARMISD::SUBE expects a carry not a borrow like ISD::USUBO_CARRY so we
6896 // have to invert the carry first.
6897 SDValue InvCarry = valueToCarryFlag(Carry, DAG, true);
6898
6899 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
6900 SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, InvCarry);
6901
6902 SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
6903 SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
6904 SDValue ARMcc = DAG.getConstant(
6905 IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32);
6906 return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc,
6907 Cmp.getValue(1));
6908}
6909
6910/// isVMOVModifiedImm - Check if the specified splat value corresponds to a
6911/// valid vector constant for a NEON or MVE instruction with a "modified
6912/// immediate" operand (e.g., VMOV). If so, return the encoded value.
6913static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
6914 unsigned SplatBitSize, SelectionDAG &DAG,
6915 const SDLoc &dl, EVT &VT, EVT VectorVT,
6916 VMOVModImmType type) {
6917 unsigned OpCmode, Imm;
6918 bool is128Bits = VectorVT.is128BitVector();
6919
6920 // SplatBitSize is set to the smallest size that splats the vector, so a
6921 // zero vector will always have SplatBitSize == 8. However, NEON modified
6922 // immediate instructions others than VMOV do not support the 8-bit encoding
6923 // of a zero vector, and the default encoding of zero is supposed to be the
6924 // 32-bit version.
6925 if (SplatBits == 0)
6926 SplatBitSize = 32;
6927
6928 switch (SplatBitSize) {
6929 case 8:
6930 if (type != VMOVModImm)
6931 return SDValue();
6932 // Any 1-byte value is OK. Op=0, Cmode=1110.
6933 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big");
6934 OpCmode = 0xe;
6935 Imm = SplatBits;
6936 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6937 break;
6938
6939 case 16:
6940 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
6941 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6942 if ((SplatBits & ~0xff) == 0) {
6943 // Value = 0x00nn: Op=x, Cmode=100x.
6944 OpCmode = 0x8;
6945 Imm = SplatBits;
6946 break;
6947 }
6948 if ((SplatBits & ~0xff00) == 0) {
6949 // Value = 0xnn00: Op=x, Cmode=101x.
6950 OpCmode = 0xa;
6951 Imm = SplatBits >> 8;
6952 break;
6953 }
6954 return SDValue();
6955
6956 case 32:
6957 // NEON's 32-bit VMOV supports splat values where:
6958 // * only one byte is nonzero, or
6959 // * the least significant byte is 0xff and the second byte is nonzero, or
6960 // * the least significant 2 bytes are 0xff and the third is nonzero.
6961 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
6962 if ((SplatBits & ~0xff) == 0) {
6963 // Value = 0x000000nn: Op=x, Cmode=000x.
6964 OpCmode = 0;
6965 Imm = SplatBits;
6966 break;
6967 }
6968 if ((SplatBits & ~0xff00) == 0) {
6969 // Value = 0x0000nn00: Op=x, Cmode=001x.
6970 OpCmode = 0x2;
6971 Imm = SplatBits >> 8;
6972 break;
6973 }
6974 if ((SplatBits & ~0xff0000) == 0) {
6975 // Value = 0x00nn0000: Op=x, Cmode=010x.
6976 OpCmode = 0x4;
6977 Imm = SplatBits >> 16;
6978 break;
6979 }
6980 if ((SplatBits & ~0xff000000) == 0) {
6981 // Value = 0xnn000000: Op=x, Cmode=011x.
6982 OpCmode = 0x6;
6983 Imm = SplatBits >> 24;
6984 break;
6985 }
6986
6987 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC
6988 if (type == OtherModImm) return SDValue();
6989
6990 if ((SplatBits & ~0xffff) == 0 &&
6991 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
6992 // Value = 0x0000nnff: Op=x, Cmode=1100.
6993 OpCmode = 0xc;
6994 Imm = SplatBits >> 8;
6995 break;
6996 }
6997
6998 // cmode == 0b1101 is not supported for MVE VMVN
6999 if (type == MVEVMVNModImm)
7000 return SDValue();
7001
7002 if ((SplatBits & ~0xffffff) == 0 &&
7003 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7004 // Value = 0x00nnffff: Op=x, Cmode=1101.
7005 OpCmode = 0xd;
7006 Imm = SplatBits >> 16;
7007 break;
7008 }
7009
7010 // Note: there are a few 32-bit splat values (specifically: 00ffff00,
7011 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
7012 // VMOV.I32. A (very) minor optimization would be to replicate the value
7013 // and fall through here to test for a valid 64-bit splat. But, then the
7014 // caller would also need to check and handle the change in size.
7015 return SDValue();
7016
7017 case 64: {
7018 if (type != VMOVModImm)
7019 return SDValue();
7020 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
7021 uint64_t BitMask = 0xff;
7022 unsigned ImmMask = 1;
7023 Imm = 0;
7024 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7025 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7026 Imm |= ImmMask;
7027 } else if ((SplatBits & BitMask) != 0) {
7028 return SDValue();
7029 }
7030 BitMask <<= 8;
7031 ImmMask <<= 1;
7032 }
7033
7034 // Op=1, Cmode=1110.
7035 OpCmode = 0x1e;
7036 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7037 break;
7038 }
7039
7040 default:
7041 llvm_unreachable("unexpected size for isVMOVModifiedImm");
7042 }
7043
7044 unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode, Imm);
7045 return DAG.getTargetConstant(EncodedVal, dl, MVT::i32);
7046}
7047
7048SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
7049 const ARMSubtarget *ST) const {
7050 EVT VT = Op.getValueType();
7051 bool IsDouble = (VT == MVT::f64);
7052 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Op);
7053 const APFloat &FPVal = CFP->getValueAPF();
7054
7055 // Prevent floating-point constants from using literal loads
7056 // when execute-only is enabled.
7057 if (ST->genExecuteOnly()) {
7058 // We shouldn't trigger this for v6m execute-only
7059 assert((!ST->isThumb1Only() || ST->hasV8MBaselineOps()) &&
7060 "Unexpected architecture");
7061
7062 // If we can represent the constant as an immediate, don't lower it
7063 if (isFPImmLegal(FPVal, VT))
7064 return Op;
7065 // Otherwise, construct as integer, and move to float register
7066 APInt INTVal = FPVal.bitcastToAPInt();
7067 SDLoc DL(CFP);
7068 switch (VT.getSimpleVT().SimpleTy) {
7069 default:
7070 llvm_unreachable("Unknown floating point type!");
7071 break;
7072 case MVT::f64: {
7073 SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32);
7074 SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32);
7075 return DAG.getNode(ARMISD::VMOVDRR, DL, MVT::f64, Lo, Hi);
7076 }
7077 case MVT::f32:
7078 return DAG.getNode(ARMISD::VMOVsr, DL, VT,
7079 DAG.getConstant(INTVal, DL, MVT::i32));
7080 }
7081 }
7082
7083 if (!ST->hasVFP3Base())
7084 return SDValue();
7085
7086 // Use the default (constant pool) lowering for double constants when we have
7087 // an SP-only FPU
7088 if (IsDouble && !Subtarget->hasFP64())
7089 return SDValue();
7090
7091 // Try splatting with a VMOV.f32...
7092 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPVal) : ARM_AM::getFP32Imm(FPVal);
7093
7094 if (ImmVal != -1) {
7095 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) {
7096 // We have code in place to select a valid ConstantFP already, no need to
7097 // do any mangling.
7098 return Op;
7099 }
7100
7101 // It's a float and we are trying to use NEON operations where
7102 // possible. Lower it to a splat followed by an extract.
7103 SDLoc DL(Op);
7104 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
7105 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
7106 NewVal);
7107 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant,
7108 DAG.getConstant(0, DL, MVT::i32));
7109 }
7110
7111 // The rest of our options are NEON only, make sure that's allowed before
7112 // proceeding..
7113 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP()))
7114 return SDValue();
7115
7116 EVT VMovVT;
7117 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue();
7118
7119 // It wouldn't really be worth bothering for doubles except for one very
7120 // important value, which does happen to match: 0.0. So make sure we don't do
7121 // anything stupid.
7122 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7123 return SDValue();
7124
7125 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
7126 SDValue NewVal = isVMOVModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
7127 VMovVT, VT, VMOVModImm);
7128 if (NewVal != SDValue()) {
7129 SDLoc DL(Op);
7130 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
7131 NewVal);
7132 if (IsDouble)
7133 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
7134
7135 // It's a float: cast and extract a vector element.
7136 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
7137 VecConstant);
7138 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
7139 DAG.getConstant(0, DL, MVT::i32));
7140 }
7141
7142 // Finally, try a VMVN.i32
7143 NewVal = isVMOVModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT,
7144 VT, VMVNModImm);
7145 if (NewVal != SDValue()) {
7146 SDLoc DL(Op);
7147 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
7148
7149 if (IsDouble)
7150 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
7151
7152 // It's a float: cast and extract a vector element.
7153 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
7154 VecConstant);
7155 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
7156 DAG.getConstant(0, DL, MVT::i32));
7157 }
7158
7159 return SDValue();
7160}
7161
7162// check if an VEXT instruction can handle the shuffle mask when the
7163// vector sources of the shuffle are the same.
7164static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
7165 unsigned NumElts = VT.getVectorNumElements();
7166
7167 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7168 if (M[0] < 0)
7169 return false;
7170
7171 Imm = M[0];
7172
7173 // If this is a VEXT shuffle, the immediate value is the index of the first
7174 // element. The other shuffle indices must be the successive elements after
7175 // the first one.
7176 unsigned ExpectedElt = Imm;
7177 for (unsigned i = 1; i < NumElts; ++i) {
7178 // Increment the expected index. If it wraps around, just follow it
7179 // back to index zero and keep going.
7180 ++ExpectedElt;
7181 if (ExpectedElt == NumElts)
7182 ExpectedElt = 0;
7183
7184 if (M[i] < 0) continue; // ignore UNDEF indices
7185 if (ExpectedElt != static_cast<unsigned>(M[i]))
7186 return false;
7187 }
7188
7189 return true;
7190}
7191
7192static bool isVEXTMask(ArrayRef<int> M, EVT VT,
7193 bool &ReverseVEXT, unsigned &Imm) {
7194 unsigned NumElts = VT.getVectorNumElements();
7195 ReverseVEXT = false;
7196
7197 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7198 if (M[0] < 0)
7199 return false;
7200
7201 Imm = M[0];
7202
7203 // If this is a VEXT shuffle, the immediate value is the index of the first
7204 // element. The other shuffle indices must be the successive elements after
7205 // the first one.
7206 unsigned ExpectedElt = Imm;
7207 for (unsigned i = 1; i < NumElts; ++i) {
7208 // Increment the expected index. If it wraps around, it may still be
7209 // a VEXT but the source vectors must be swapped.
7210 ExpectedElt += 1;
7211 if (ExpectedElt == NumElts * 2) {
7212 ExpectedElt = 0;
7213 ReverseVEXT = true;
7214 }
7215
7216 if (M[i] < 0) continue; // ignore UNDEF indices
7217 if (ExpectedElt != static_cast<unsigned>(M[i]))
7218 return false;
7219 }
7220
7221 // Adjust the index value if the source operands will be swapped.
7222 if (ReverseVEXT)
7223 Imm -= NumElts;
7224
7225 return true;
7226}
7227
7228static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
7229 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
7230 // range, then 0 is placed into the resulting vector. So pretty much any mask
7231 // of 8 elements can work here.
7232 return VT == MVT::v8i8 && M.size() == 8;
7233}
7234
7235/// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
7236/// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
7237static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
7238 unsigned &WhichResult,
7239 bool &isV_UNDEF) {
7240 isV_UNDEF = false;
7241 if (isVTRNMask(ShuffleMask, VT, WhichResult))
7242 return ARMISD::VTRN;
7243 if (isVUZPMask(ShuffleMask, VT, WhichResult))
7244 return ARMISD::VUZP;
7245 if (isVZIPMask(ShuffleMask, VT, WhichResult))
7246 return ARMISD::VZIP;
7247
7248 isV_UNDEF = true;
7249 if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult))
7250 return ARMISD::VTRN;
7251 if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult))
7252 return ARMISD::VUZP;
7253 if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult))
7254 return ARMISD::VZIP;
7255
7256 return 0;
7257}
7258
7259/// \return true if this is a reverse operation on an vector.
7260static bool isReverseMask(ArrayRef<int> M, EVT VT) {
7261 unsigned NumElts = VT.getVectorNumElements();
7262 // Make sure the mask has the right size.
7263 if (NumElts != M.size())
7264 return false;
7265
7266 // Look for <15, ..., 3, -1, 1, 0>.
7267 for (unsigned i = 0; i != NumElts; ++i)
7268 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
7269 return false;
7270
7271 return true;
7272}
7273
7274static bool isTruncMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7275 unsigned NumElts = VT.getVectorNumElements();
7276 // Make sure the mask has the right size.
7277 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7278 return false;
7279
7280 // Half-width truncation patterns (e.g. v4i32 -> v8i16):
7281 // !Top && SingleSource: <0, 2, 4, 6, 0, 2, 4, 6>
7282 // !Top && !SingleSource: <0, 2, 4, 6, 8, 10, 12, 14>
7283 // Top && SingleSource: <1, 3, 5, 7, 1, 3, 5, 7>
7284 // Top && !SingleSource: <1, 3, 5, 7, 9, 11, 13, 15>
7285 int Ofs = Top ? 1 : 0;
7286 int Upper = SingleSource ? 0 : NumElts;
7287 for (int i = 0, e = NumElts / 2; i != e; ++i) {
7288 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7289 return false;
7290 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs + Upper)
7291 return false;
7292 }
7293 return true;
7294}
7295
7296static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7297 unsigned NumElts = VT.getVectorNumElements();
7298 // Make sure the mask has the right size.
7299 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7300 return false;
7301
7302 // If Top
7303 // Look for <0, N, 2, N+2, 4, N+4, ..>.
7304 // This inserts Input2 into Input1
7305 // else if not Top
7306 // Look for <0, N+1, 2, N+3, 4, N+5, ..>
7307 // This inserts Input1 into Input2
7308 unsigned Offset = Top ? 0 : 1;
7309 unsigned N = SingleSource ? 0 : NumElts;
7310 for (unsigned i = 0; i < NumElts; i += 2) {
7311 if (M[i] >= 0 && M[i] != (int)i)
7312 return false;
7313 if (M[i + 1] >= 0 && M[i + 1] != (int)(N + i + Offset))
7314 return false;
7315 }
7316
7317 return true;
7318}
7319
7320static bool isVMOVNTruncMask(ArrayRef<int> M, EVT ToVT, bool rev) {
7321 unsigned NumElts = ToVT.getVectorNumElements();
7322 if (NumElts != M.size())
7323 return false;
7324
7325 // Test if the Trunc can be convertible to a VMOVN with this shuffle. We are
7326 // looking for patterns of:
7327 // !rev: 0 N/2 1 N/2+1 2 N/2+2 ...
7328 // rev: N/2 0 N/2+1 1 N/2+2 2 ...
7329
7330 unsigned Off0 = rev ? NumElts / 2 : 0;
7331 unsigned Off1 = rev ? 0 : NumElts / 2;
7332 for (unsigned i = 0; i < NumElts; i += 2) {
7333 if (M[i] >= 0 && M[i] != (int)(Off0 + i / 2))
7334 return false;
7335 if (M[i + 1] >= 0 && M[i + 1] != (int)(Off1 + i / 2))
7336 return false;
7337 }
7338
7339 return true;
7340}
7341
7342// Reconstruct an MVE VCVT from a BuildVector of scalar fptrunc, all extracted
7343// from a pair of inputs. For example:
7344// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7345// FP_ROUND(EXTRACT_ELT(Y, 0),
7346// FP_ROUND(EXTRACT_ELT(X, 1),
7347// FP_ROUND(EXTRACT_ELT(Y, 1), ...)
7349 const ARMSubtarget *ST) {
7350 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7351 if (!ST->hasMVEFloatOps())
7352 return SDValue();
7353
7354 SDLoc dl(BV);
7355 EVT VT = BV.getValueType();
7356 if (VT != MVT::v8f16)
7357 return SDValue();
7358
7359 // We are looking for a buildvector of fptrunc elements, where all the
7360 // elements are interleavingly extracted from two sources. Check the first two
7361 // items are valid enough and extract some info from them (they are checked
7362 // properly in the loop below).
7363 if (BV.getOperand(0).getOpcode() != ISD::FP_ROUND ||
7366 return SDValue();
7367 if (BV.getOperand(1).getOpcode() != ISD::FP_ROUND ||
7370 return SDValue();
7371 SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
7372 SDValue Op1 = BV.getOperand(1).getOperand(0).getOperand(0);
7373 if (Op0.getValueType() != MVT::v4f32 || Op1.getValueType() != MVT::v4f32)
7374 return SDValue();
7375
7376 // Check all the values in the BuildVector line up with our expectations.
7377 for (unsigned i = 1; i < 4; i++) {
7378 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7379 return Trunc.getOpcode() == ISD::FP_ROUND &&
7381 Trunc.getOperand(0).getOperand(0) == Op &&
7382 Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
7383 };
7384 if (!Check(BV.getOperand(i * 2 + 0), Op0, i))
7385 return SDValue();
7386 if (!Check(BV.getOperand(i * 2 + 1), Op1, i))
7387 return SDValue();
7388 }
7389
7390 SDValue N1 = DAG.getNode(ARMISD::VCVTN, dl, VT, DAG.getUNDEF(VT), Op0,
7391 DAG.getConstant(0, dl, MVT::i32));
7392 return DAG.getNode(ARMISD::VCVTN, dl, VT, N1, Op1,
7393 DAG.getConstant(1, dl, MVT::i32));
7394}
7395
7396// Reconstruct an MVE VCVT from a BuildVector of scalar fpext, all extracted
7397// from a single input on alternating lanes. For example:
7398// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7399// FP_ROUND(EXTRACT_ELT(X, 2),
7400// FP_ROUND(EXTRACT_ELT(X, 4), ...)
7402 const ARMSubtarget *ST) {
7403 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7404 if (!ST->hasMVEFloatOps())
7405 return SDValue();
7406
7407 SDLoc dl(BV);
7408 EVT VT = BV.getValueType();
7409 if (VT != MVT::v4f32)
7410 return SDValue();
7411
7412 // We are looking for a buildvector of fptext elements, where all the
7413 // elements are alternating lanes from a single source. For example <0,2,4,6>
7414 // or <1,3,5,7>. Check the first two items are valid enough and extract some
7415 // info from them (they are checked properly in the loop below).
7416 if (BV.getOperand(0).getOpcode() != ISD::FP_EXTEND ||
7418 return SDValue();
7419 SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
7421 if (Op0.getValueType() != MVT::v8f16 || (Offset != 0 && Offset != 1))
7422 return SDValue();
7423
7424 // Check all the values in the BuildVector line up with our expectations.
7425 for (unsigned i = 1; i < 4; i++) {
7426 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7427 return Trunc.getOpcode() == ISD::FP_EXTEND &&
7429 Trunc.getOperand(0).getOperand(0) == Op &&
7430 Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
7431 };
7432 if (!Check(BV.getOperand(i), Op0, 2 * i + Offset))
7433 return SDValue();
7434 }
7435
7436 return DAG.getNode(ARMISD::VCVTL, dl, VT, Op0,
7437 DAG.getConstant(Offset, dl, MVT::i32));
7438}
7439
7440// If N is an integer constant that can be moved into a register in one
7441// instruction, return an SDValue of such a constant (will become a MOV
7442// instruction). Otherwise return null.
7444 const ARMSubtarget *ST, const SDLoc &dl) {
7445 uint64_t Val;
7446 if (!isa<ConstantSDNode>(N))
7447 return SDValue();
7448 Val = N->getAsZExtVal();
7449
7450 if (ST->isThumb1Only()) {
7451 if (Val <= 255 || ~Val <= 255)
7452 return DAG.getConstant(Val, dl, MVT::i32);
7453 } else {
7454 if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1)
7455 return DAG.getConstant(Val, dl, MVT::i32);
7456 }
7457 return SDValue();
7458}
7459
7461 const ARMSubtarget *ST) {
7462 SDLoc dl(Op);
7463 EVT VT = Op.getValueType();
7464
7465 assert(ST->hasMVEIntegerOps() && "LowerBUILD_VECTOR_i1 called without MVE!");
7466
7467 unsigned NumElts = VT.getVectorNumElements();
7468 unsigned BoolMask;
7469 unsigned BitsPerBool;
7470 if (NumElts == 2) {
7471 BitsPerBool = 8;
7472 BoolMask = 0xff;
7473 } else if (NumElts == 4) {
7474 BitsPerBool = 4;
7475 BoolMask = 0xf;
7476 } else if (NumElts == 8) {
7477 BitsPerBool = 2;
7478 BoolMask = 0x3;
7479 } else if (NumElts == 16) {
7480 BitsPerBool = 1;
7481 BoolMask = 0x1;
7482 } else
7483 return SDValue();
7484
7485 // If this is a single value copied into all lanes (a splat), we can just sign
7486 // extend that single value
7487 SDValue FirstOp = Op.getOperand(0);
7488 if (!isa<ConstantSDNode>(FirstOp) &&
7489 llvm::all_of(llvm::drop_begin(Op->ops()), [&FirstOp](const SDUse &U) {
7490 return U.get().isUndef() || U.get() == FirstOp;
7491 })) {
7492 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, FirstOp,
7493 DAG.getValueType(MVT::i1));
7494 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), Ext);
7495 }
7496
7497 // First create base with bits set where known
7498 unsigned Bits32 = 0;
7499 for (unsigned i = 0; i < NumElts; ++i) {
7500 SDValue V = Op.getOperand(i);
7501 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7502 continue;
7503 bool BitSet = V.isUndef() ? false : V->getAsZExtVal();
7504 if (BitSet)
7505 Bits32 |= BoolMask << (i * BitsPerBool);
7506 }
7507
7508 // Add in unknown nodes
7509 SDValue Base = DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
7510 DAG.getConstant(Bits32, dl, MVT::i32));
7511 for (unsigned i = 0; i < NumElts; ++i) {
7512 SDValue V = Op.getOperand(i);
7513 if (isa<ConstantSDNode>(V) || V.isUndef())
7514 continue;
7515 Base = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Base, V,
7516 DAG.getConstant(i, dl, MVT::i32));
7517 }
7518
7519 return Base;
7520}
7521
7523 const ARMSubtarget *ST) {
7524 if (!ST->hasMVEIntegerOps())
7525 return SDValue();
7526
7527 // We are looking for a buildvector where each element is Op[0] + i*N
7528 EVT VT = Op.getValueType();
7529 SDValue Op0 = Op.getOperand(0);
7530 unsigned NumElts = VT.getVectorNumElements();
7531
7532 // Get the increment value from operand 1
7533 SDValue Op1 = Op.getOperand(1);
7534 if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(0) != Op0 ||
7536 return SDValue();
7537 unsigned N = Op1.getConstantOperandVal(1);
7538 if (N != 1 && N != 2 && N != 4 && N != 8)
7539 return SDValue();
7540
7541 // Check that each other operand matches
7542 for (unsigned I = 2; I < NumElts; I++) {
7543 SDValue OpI = Op.getOperand(I);
7544 if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(0) != Op0 ||
7546 OpI.getConstantOperandVal(1) != I * N)
7547 return SDValue();
7548 }
7549
7550 SDLoc DL(Op);
7551 return DAG.getNode(ARMISD::VIDUP, DL, DAG.getVTList(VT, MVT::i32), Op0,
7552 DAG.getConstant(N, DL, MVT::i32));
7553}
7554
7555// Returns true if the operation N can be treated as qr instruction variant at
7556// operand Op.
7557static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op) {
7558 switch (N->getOpcode()) {
7559 case ISD::ADD:
7560 case ISD::MUL:
7561 case ISD::SADDSAT:
7562 case ISD::UADDSAT:
7563 case ISD::AVGFLOORS:
7564 case ISD::AVGFLOORU:
7565 return true;
7566 case ISD::SUB:
7567 case ISD::SSUBSAT:
7568 case ISD::USUBSAT:
7569 return N->getOperand(1).getNode() == Op;
7571 switch (N->getConstantOperandVal(0)) {
7572 case Intrinsic::arm_mve_add_predicated:
7573 case Intrinsic::arm_mve_mul_predicated:
7574 case Intrinsic::arm_mve_qadd_predicated:
7575 case Intrinsic::arm_mve_vhadd:
7576 case Intrinsic::arm_mve_hadd_predicated:
7577 case Intrinsic::arm_mve_vqdmulh:
7578 case Intrinsic::arm_mve_qdmulh_predicated:
7579 case Intrinsic::arm_mve_vqrdmulh:
7580 case Intrinsic::arm_mve_qrdmulh_predicated:
7581 case Intrinsic::arm_mve_vqdmull:
7582 case Intrinsic::arm_mve_vqdmull_predicated:
7583 return true;
7584 case Intrinsic::arm_mve_sub_predicated:
7585 case Intrinsic::arm_mve_qsub_predicated:
7586 case Intrinsic::arm_mve_vhsub:
7587 case Intrinsic::arm_mve_hsub_predicated:
7588 return N->getOperand(2).getNode() == Op;
7589 default:
7590 return false;
7591 }
7592 default:
7593 return false;
7594 }
7595}
7596
7597// If this is a case we can't handle, return null and let the default
7598// expansion code take care of it.
7599SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7600 const ARMSubtarget *ST) const {
7601 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
7602 SDLoc dl(Op);
7603 EVT VT = Op.getValueType();
7604
7605 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
7606 return LowerBUILD_VECTOR_i1(Op, DAG, ST);
7607
7608 if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
7609 return R;
7610
7611 APInt SplatBits, SplatUndef;
7612 unsigned SplatBitSize;
7613 bool HasAnyUndefs;
7614 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7615 if (SplatUndef.isAllOnes())
7616 return DAG.getUNDEF(VT);
7617
7618 // If all the users of this constant splat are qr instruction variants,
7619 // generate a vdup of the constant.
7620 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize &&
7621 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7622 all_of(BVN->users(),
7623 [BVN](const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7624 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7625 : SplatBitSize == 16 ? MVT::v8i16
7626 : MVT::v16i8;
7627 SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32);
7628 SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const);
7629 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
7630 }
7631
7632 if ((ST->hasNEON() && SplatBitSize <= 64) ||
7633 (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7634 // Check if an immediate VMOV works.
7635 EVT VmovVT;
7636 SDValue Val =
7637 isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
7638 SplatBitSize, DAG, dl, VmovVT, VT, VMOVModImm);
7639
7640 if (Val.getNode()) {
7641 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
7642 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vmov);
7643 }
7644
7645 // Try an immediate VMVN.
7646 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7647 Val = isVMOVModifiedImm(
7648 NegatedImm, SplatUndef.getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7649 VT, ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm);
7650 if (Val.getNode()) {
7651 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
7652 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vmov);
7653 }
7654
7655 // Use vmov.f32 to materialize other v2f32 and v4f32 splats.
7656 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7657 int ImmVal = ARM_AM::getFP32Imm(SplatBits);
7658 if (ImmVal != -1) {
7659 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
7660 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val);
7661 }
7662 }
7663
7664 // If we are under MVE, generate a VDUP(constant), bitcast to the original
7665 // type.
7666 if (ST->hasMVEIntegerOps() &&
7667 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7668 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7669 : SplatBitSize == 16 ? MVT::v8i16
7670 : MVT::v16i8;
7671 SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32);
7672 SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const);
7673 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
7674 }
7675 }
7676 }
7677
7678 // Scan through the operands to see if only one value is used.
7679 //
7680 // As an optimisation, even if more than one value is used it may be more
7681 // profitable to splat with one value then change some lanes.
7682 //
7683 // Heuristically we decide to do this if the vector has a "dominant" value,
7684 // defined as splatted to more than half of the lanes.
7685 unsigned NumElts = VT.getVectorNumElements();
7686 bool isOnlyLowElement = true;
7687 bool usesOnlyOneValue = true;
7688 bool hasDominantValue = false;
7689 bool isConstant = true;
7690
7691 // Map of the number of times a particular SDValue appears in the
7692 // element list.
7693 DenseMap<SDValue, unsigned> ValueCounts;
7694 SDValue Value;
7695 for (unsigned i = 0; i < NumElts; ++i) {
7696 SDValue V = Op.getOperand(i);
7697 if (V.isUndef())
7698 continue;
7699 if (i > 0)
7700 isOnlyLowElement = false;
7702 isConstant = false;
7703
7704 unsigned &Count = ValueCounts[V];
7705
7706 // Is this value dominant? (takes up more than half of the lanes)
7707 if (++Count > (NumElts / 2)) {
7708 hasDominantValue = true;
7709 Value = V;
7710 }
7711 }
7712 if (ValueCounts.size() != 1)
7713 usesOnlyOneValue = false;
7714 if (!Value.getNode() && !ValueCounts.empty())
7715 Value = ValueCounts.begin()->first;
7716
7717 if (ValueCounts.empty())
7718 return DAG.getUNDEF(VT);
7719
7720 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR.
7721 // Keep going if we are hitting this case.
7722 if (isOnlyLowElement && !ISD::isNormalLoad(Value.getNode()) &&
7723 (VT != MVT::v8f16 || ST->hasFullFP16()))
7724 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
7725
7726 unsigned EltSize = VT.getScalarSizeInBits();
7727
7728 // Use VDUP for non-constant splats. For f32 constant splats, reduce to
7729 // i32 and try again.
7730 if (hasDominantValue && EltSize <= 32) {
7731 if (!isConstant) {
7732 SDValue N;
7733
7734 // If we are VDUPing a value that comes directly from a vector, that will
7735 // cause an unnecessary move to and from a GPR, where instead we could
7736 // just use VDUPLANE. We can only do this if the lane being extracted
7737 // is at a constant index, as the VDUP from lane instructions only have
7738 // constant-index forms.
7739 ConstantSDNode *constIndex;
7740 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7741 (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) {
7742 // We need to create a new undef vector to use for the VDUPLANE if the
7743 // size of the vector from which we get the value is different than the
7744 // size of the vector that we need to create. We will insert the element
7745 // such that the register coalescer will remove unnecessary copies.
7746 if (VT != Value->getOperand(0).getValueType()) {
7747 unsigned index = constIndex->getAPIntValue().getLimitedValue() %
7749 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
7750 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT),
7751 Value, DAG.getConstant(index, dl, MVT::i32)),
7752 DAG.getConstant(index, dl, MVT::i32));
7753 } else
7754 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
7755 Value->getOperand(0), Value->getOperand(1));
7756 } else
7757 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value);
7758
7759 if (!usesOnlyOneValue) {
7760 // The dominant value was splatted as 'N', but we now have to insert
7761 // all differing elements.
7762 for (unsigned I = 0; I < NumElts; ++I) {
7763 if (Op.getOperand(I) == Value)
7764 continue;
7766 Ops.push_back(N);
7767 Ops.push_back(Op.getOperand(I));
7768 Ops.push_back(DAG.getConstant(I, dl, MVT::i32));
7769 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops);
7770 }
7771 }
7772 return N;
7773 }
7776 MVT FVT = VT.getVectorElementType().getSimpleVT();
7777 assert(FVT == MVT::f32 || FVT == MVT::f16);
7778 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
7779 for (unsigned i = 0; i < NumElts; ++i)
7780 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, IVT,
7781 Op.getOperand(i)));
7782 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), IVT, NumElts);
7783 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
7784 Val = LowerBUILD_VECTOR(Val, DAG, ST);
7785 if (Val.getNode())
7786 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
7787 }
7788 if (usesOnlyOneValue) {
7789 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
7790 if (isConstant && Val.getNode())
7791 return DAG.getNode(ARMISD::VDUP, dl, VT, Val);
7792 }
7793 }
7794
7795 // If all elements are constants and the case above didn't get hit, fall back
7796 // to the default expansion, which will generate a load from the constant
7797 // pool.
7798 if (isConstant)
7799 return SDValue();
7800
7801 // Reconstruct the BUILDVECTOR to one of the legal shuffles (such as vext and
7802 // vmovn). Empirical tests suggest this is rarely worth it for vectors of
7803 // length <= 2.
7804 if (NumElts >= 4)
7805 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
7806 return shuffle;
7807
7808 // Attempt to turn a buildvector of scalar fptrunc's or fpext's back into
7809 // VCVT's
7810 if (SDValue VCVT = LowerBuildVectorOfFPTrunc(Op, DAG, Subtarget))
7811 return VCVT;
7812 if (SDValue VCVT = LowerBuildVectorOfFPExt(Op, DAG, Subtarget))
7813 return VCVT;
7814
7815 if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
7816 // If we haven't found an efficient lowering, try splitting a 128-bit vector
7817 // into two 64-bit vectors; we might discover a better way to lower it.
7818 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts);
7819 EVT ExtVT = VT.getVectorElementType();
7820 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElts / 2);
7821 SDValue Lower = DAG.getBuildVector(HVT, dl, ArrayRef(&Ops[0], NumElts / 2));
7822 if (Lower.getOpcode() == ISD::BUILD_VECTOR)
7823 Lower = LowerBUILD_VECTOR(Lower, DAG, ST);
7824 SDValue Upper =
7825 DAG.getBuildVector(HVT, dl, ArrayRef(&Ops[NumElts / 2], NumElts / 2));
7826 if (Upper.getOpcode() == ISD::BUILD_VECTOR)
7827 Upper = LowerBUILD_VECTOR(Upper, DAG, ST);
7828 if (Lower && Upper)
7829 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper);
7830 }
7831
7832 // Vectors with 32- or 64-bit elements can be built by directly assigning
7833 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands
7834 // will be legalized.
7835 if (EltSize >= 32) {
7836 // Do the expansion with floating-point types, since that is what the VFP
7837 // registers are defined to use, and since i64 is not legal.
7838 EVT EltVT = EVT::getFloatingPointVT(EltSize);
7839 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
7841 for (unsigned i = 0; i < NumElts; ++i)
7842 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i)));
7843 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
7844 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
7845 }
7846
7847 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
7848 // know the default expansion would otherwise fall back on something even
7849 // worse. For a vector with one or two non-undef values, that's
7850 // scalar_to_vector for the elements followed by a shuffle (provided the
7851 // shuffle is valid for the target) and materialization element by element
7852 // on the stack followed by a load for everything else.
7853 if ((!isConstant && !usesOnlyOneValue) ||
7854 (VT == MVT::v8f16 && !ST->hasFullFP16())) {
7855 SDValue Vec = DAG.getUNDEF(VT);
7856 for (unsigned i = 0 ; i < NumElts; ++i) {
7857 SDValue V = Op.getOperand(i);
7858 if (V.isUndef())
7859 continue;
7860 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
7861 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
7862 }
7863 return Vec;
7864 }
7865
7866 return SDValue();
7867}
7868
7869// Gather data to see if the operation can be modelled as a
7870// shuffle in combination with VEXTs.
7871SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
7872 SelectionDAG &DAG) const {
7873 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7874 SDLoc dl(Op);
7875 EVT VT = Op.getValueType();
7876 unsigned NumElts = VT.getVectorNumElements();
7877
7878 struct ShuffleSourceInfo {
7879 SDValue Vec;
7880 unsigned MinElt = std::numeric_limits<unsigned>::max();
7881 unsigned MaxElt = 0;
7882
7883 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
7884 // be compatible with the shuffle we intend to construct. As a result
7885 // ShuffleVec will be some sliding window into the original Vec.
7886 SDValue ShuffleVec;
7887
7888 // Code should guarantee that element i in Vec starts at element "WindowBase
7889 // + i * WindowScale in ShuffleVec".
7890 int WindowBase = 0;
7891 int WindowScale = 1;
7892
7893 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
7894
7895 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
7896 };
7897
7898 // First gather all vectors used as an immediate source for this BUILD_VECTOR
7899 // node.
7901 for (unsigned i = 0; i < NumElts; ++i) {
7902 SDValue V = Op.getOperand(i);
7903 if (V.isUndef())
7904 continue;
7905 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
7906 // A shuffle can only come from building a vector from various
7907 // elements of other vectors.
7908 return SDValue();
7909 } else if (!isa<ConstantSDNode>(V.getOperand(1))) {
7910 // Furthermore, shuffles require a constant mask, whereas extractelts
7911 // accept variable indices.
7912 return SDValue();
7913 }
7914
7915 // Add this element source to the list if it's not already there.
7916 SDValue SourceVec = V.getOperand(0);
7917 auto Source = llvm::find(Sources, SourceVec);
7918 if (Source == Sources.end())
7919 Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec));
7920
7921 // Update the minimum and maximum lane number seen.
7922 unsigned EltNo = V.getConstantOperandVal(1);
7923 Source->MinElt = std::min(Source->MinElt, EltNo);
7924 Source->MaxElt = std::max(Source->MaxElt, EltNo);
7925 }
7926
7927 // Currently only do something sane when at most two source vectors
7928 // are involved.
7929 if (Sources.size() > 2)
7930 return SDValue();
7931
7932 // Find out the smallest element size among result and two sources, and use
7933 // it as element size to build the shuffle_vector.
7934 EVT SmallestEltTy = VT.getVectorElementType();
7935 for (auto &Source : Sources) {
7936 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
7937 if (SrcEltTy.bitsLT(SmallestEltTy))
7938 SmallestEltTy = SrcEltTy;
7939 }
7940 unsigned ResMultiplier =
7941 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits();
7942 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits();
7943 EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts);
7944
7945 // If the source vector is too wide or too narrow, we may nevertheless be able
7946 // to construct a compatible shuffle either by concatenating it with UNDEF or
7947 // extracting a suitable range of elements.
7948 for (auto &Src : Sources) {
7949 EVT SrcVT = Src.ShuffleVec.getValueType();
7950
7951 uint64_t SrcVTSize = SrcVT.getFixedSizeInBits();
7952 uint64_t VTSize = VT.getFixedSizeInBits();
7953 if (SrcVTSize == VTSize)
7954 continue;
7955
7956 // This stage of the search produces a source with the same element type as
7957 // the original, but with a total width matching the BUILD_VECTOR output.
7958 EVT EltVT = SrcVT.getVectorElementType();
7959 unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
7960 EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
7961
7962 if (SrcVTSize < VTSize) {
7963 if (2 * SrcVTSize != VTSize)
7964 return SDValue();
7965 // We can pad out the smaller vector for free, so if it's part of a
7966 // shuffle...
7967 Src.ShuffleVec =
7968 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
7969 DAG.getUNDEF(Src.ShuffleVec.getValueType()));
7970 continue;
7971 }
7972
7973 if (SrcVTSize != 2 * VTSize)
7974 return SDValue();
7975
7976 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
7977 // Span too large for a VEXT to cope
7978 return SDValue();
7979 }
7980
7981 if (Src.MinElt >= NumSrcElts) {
7982 // The extraction can just take the second half
7983 Src.ShuffleVec =
7984 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
7985 DAG.getConstant(NumSrcElts, dl, MVT::i32));
7986 Src.WindowBase = -NumSrcElts;
7987 } else if (Src.MaxElt < NumSrcElts) {
7988 // The extraction can just take the first half
7989 Src.ShuffleVec =
7990 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
7991 DAG.getConstant(0, dl, MVT::i32));
7992 } else {
7993 // An actual VEXT is needed
7994 SDValue VEXTSrc1 =
7995 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
7996 DAG.getConstant(0, dl, MVT::i32));
7997 SDValue VEXTSrc2 =
7998 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
7999 DAG.getConstant(NumSrcElts, dl, MVT::i32));
8000
8001 Src.ShuffleVec = DAG.getNode(ARMISD::VEXT, dl, DestVT, VEXTSrc1,
8002 VEXTSrc2,
8003 DAG.getConstant(Src.MinElt, dl, MVT::i32));
8004 Src.WindowBase = -Src.MinElt;
8005 }
8006 }
8007
8008 // Another possible incompatibility occurs from the vector element types. We
8009 // can fix this by bitcasting the source vectors to the same type we intend
8010 // for the shuffle.
8011 for (auto &Src : Sources) {
8012 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8013 if (SrcEltTy == SmallestEltTy)
8014 continue;
8015 assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
8016 Src.ShuffleVec = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, ShuffleVT, Src.ShuffleVec);
8017 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits();
8018 Src.WindowBase *= Src.WindowScale;
8019 }
8020
8021 // Final check before we try to actually produce a shuffle.
8022 LLVM_DEBUG({
8023 for (auto Src : Sources)
8024 assert(Src.ShuffleVec.getValueType() == ShuffleVT);
8025 });
8026
8027 // The stars all align, our next step is to produce the mask for the shuffle.
8028 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
8029 int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
8030 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
8031 SDValue Entry = Op.getOperand(i);
8032 if (Entry.isUndef())
8033 continue;
8034
8035 auto Src = llvm::find(Sources, Entry.getOperand(0));
8036 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
8037
8038 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
8039 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
8040 // segment.
8041 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
8042 int BitsDefined = std::min(OrigEltTy.getScalarSizeInBits(),
8043 VT.getScalarSizeInBits());
8044 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8045
8046 // This source is expected to fill ResMultiplier lanes of the final shuffle,
8047 // starting at the appropriate offset.
8048 int *LaneMask = &Mask[i * ResMultiplier];
8049
8050 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8051 ExtractBase += NumElts * (Src - Sources.begin());
8052 for (int j = 0; j < LanesDefined; ++j)
8053 LaneMask[j] = ExtractBase + j;
8054 }
8055
8056
8057 // We can't handle more than two sources. This should have already
8058 // been checked before this point.
8059 assert(Sources.size() <= 2 && "Too many sources!");
8060
8061 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
8062 for (unsigned i = 0; i < Sources.size(); ++i)
8063 ShuffleOps[i] = Sources[i].ShuffleVec;
8064
8065 SDValue Shuffle = buildLegalVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
8066 ShuffleOps[1], Mask, DAG);
8067 if (!Shuffle)
8068 return SDValue();
8069 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Shuffle);
8070}
8071
8073 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
8082 OP_VUZPL, // VUZP, left result
8083 OP_VUZPR, // VUZP, right result
8084 OP_VZIPL, // VZIP, left result
8085 OP_VZIPR, // VZIP, right result
8086 OP_VTRNL, // VTRN, left result
8087 OP_VTRNR // VTRN, right result
8088};
8089
8090static bool isLegalMVEShuffleOp(unsigned PFEntry) {
8091 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8092 switch (OpNum) {
8093 case OP_COPY:
8094 case OP_VREV:
8095 case OP_VDUP0:
8096 case OP_VDUP1:
8097 case OP_VDUP2:
8098 case OP_VDUP3:
8099 return true;
8100 }
8101 return false;
8102}
8103
8104/// isShuffleMaskLegal - Targets can use this to indicate that they only
8105/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
8106/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
8107/// are assumed to be legal.
8109 if (VT.getVectorNumElements() == 4 &&
8110 (VT.is128BitVector() || VT.is64BitVector())) {
8111 unsigned PFIndexes[4];
8112 for (unsigned i = 0; i != 4; ++i) {
8113 if (M[i] < 0)
8114 PFIndexes[i] = 8;
8115 else
8116 PFIndexes[i] = M[i];
8117 }
8118
8119 // Compute the index in the perfect shuffle table.
8120 unsigned PFTableIndex =
8121 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8122 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8123 unsigned Cost = (PFEntry >> 30);
8124
8125 if (Cost <= 4 && (Subtarget->hasNEON() || isLegalMVEShuffleOp(PFEntry)))
8126 return true;
8127 }
8128
8129 bool ReverseVEXT, isV_UNDEF;
8130 unsigned Imm, WhichResult;
8131
8132 unsigned EltSize = VT.getScalarSizeInBits();
8133 if (EltSize >= 32 ||
8135 ShuffleVectorInst::isIdentityMask(M, M.size()) ||
8136 isVREVMask(M, VT, 64) ||
8137 isVREVMask(M, VT, 32) ||
8138 isVREVMask(M, VT, 16))
8139 return true;
8140 else if (Subtarget->hasNEON() &&
8141 (isVEXTMask(M, VT, ReverseVEXT, Imm) ||
8142 isVTBLMask(M, VT) ||
8143 isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF)))
8144 return true;
8145 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8146 isReverseMask(M, VT))
8147 return true;
8148 else if (Subtarget->hasMVEIntegerOps() &&
8149 (isVMOVNMask(M, VT, true, false) ||
8150 isVMOVNMask(M, VT, false, false) || isVMOVNMask(M, VT, true, true)))
8151 return true;
8152 else if (Subtarget->hasMVEIntegerOps() &&
8153 (isTruncMask(M, VT, false, false) ||
8154 isTruncMask(M, VT, false, true) ||
8155 isTruncMask(M, VT, true, false) || isTruncMask(M, VT, true, true)))
8156 return true;
8157 else
8158 return false;
8159}
8160
8161/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
8162/// the specified operations to build the shuffle.
8164 SDValue RHS, SelectionDAG &DAG,
8165 const SDLoc &dl) {
8166 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8167 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8168 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8169
8170 if (OpNum == OP_COPY) {
8171 if (LHSID == (1*9+2)*9+3) return LHS;
8172 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
8173 return RHS;
8174 }
8175
8176 SDValue OpLHS, OpRHS;
8177 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
8178 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
8179 EVT VT = OpLHS.getValueType();
8180
8181 switch (OpNum) {
8182 default: llvm_unreachable("Unknown shuffle opcode!");
8183 case OP_VREV:
8184 // VREV divides the vector in half and swaps within the half.
8185 if (VT.getScalarSizeInBits() == 32)
8186 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS);
8187 // vrev <4 x i16> -> VREV32
8188 if (VT.getScalarSizeInBits() == 16)
8189 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS);
8190 // vrev <4 x i8> -> VREV16
8191 assert(VT.getScalarSizeInBits() == 8);
8192 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS);
8193 case OP_VDUP0:
8194 case OP_VDUP1:
8195 case OP_VDUP2:
8196 case OP_VDUP3:
8197 return DAG.getNode(ARMISD::VDUPLANE, dl, VT,
8198 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32));
8199 case OP_VEXT1:
8200 case OP_VEXT2:
8201 case OP_VEXT3:
8202 return DAG.getNode(ARMISD::VEXT, dl, VT,
8203 OpLHS, OpRHS,
8204 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32));
8205 case OP_VUZPL:
8206 case OP_VUZPR:
8207 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
8208 OpLHS, OpRHS).getValue(OpNum-OP_VUZPL);
8209 case OP_VZIPL:
8210 case OP_VZIPR:
8211 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
8212 OpLHS, OpRHS).getValue(OpNum-OP_VZIPL);
8213 case OP_VTRNL:
8214 case OP_VTRNR:
8215 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
8216 OpLHS, OpRHS).getValue(OpNum-OP_VTRNL);
8217 }
8218}
8219
8221 ArrayRef<int> ShuffleMask,
8222 SelectionDAG &DAG) {
8223 // Check to see if we can use the VTBL instruction.
8224 SDValue V1 = Op.getOperand(0);
8225 SDValue V2 = Op.getOperand(1);
8226 SDLoc DL(Op);
8227
8228 SmallVector<SDValue, 8> VTBLMask;
8229 for (int I : ShuffleMask)
8230 VTBLMask.push_back(DAG.getSignedConstant(I, DL, MVT::i32));
8231
8232 if (V2.getNode()->isUndef())
8233 return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
8234 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
8235
8236 return DAG.getNode(ARMISD::VTBL2, DL, MVT::v8i8, V1, V2,
8237 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
8238}
8239
8241 SDLoc DL(Op);
8242 EVT VT = Op.getValueType();
8243
8244 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8245 "Expect an v8i16/v16i8 type");
8246 SDValue OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, Op.getOperand(0));
8247 // For a v16i8 type: After the VREV, we have got <7, ..., 0, 15, ..., 8>. Now,
8248 // extract the first 8 bytes into the top double word and the last 8 bytes
8249 // into the bottom double word, through a new vector shuffle that will be
8250 // turned into a VEXT on Neon, or a couple of VMOVDs on MVE.
8251 std::vector<int> NewMask;
8252 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8253 NewMask.push_back(VT.getVectorNumElements() / 2 + i);
8254 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8255 NewMask.push_back(i);
8256 return DAG.getVectorShuffle(VT, DL, OpLHS, OpLHS, NewMask);
8257}
8258
8260 switch (VT.getSimpleVT().SimpleTy) {
8261 case MVT::v2i1:
8262 return MVT::v2f64;
8263 case MVT::v4i1:
8264 return MVT::v4i32;
8265 case MVT::v8i1:
8266 return MVT::v8i16;
8267 case MVT::v16i1:
8268 return MVT::v16i8;
8269 default:
8270 llvm_unreachable("Unexpected vector predicate type");
8271 }
8272}
8273
8275 SelectionDAG &DAG) {
8276 // Converting from boolean predicates to integers involves creating a vector
8277 // of all ones or all zeroes and selecting the lanes based upon the real
8278 // predicate.
8280 DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff), dl, MVT::i32);
8281 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v16i8, AllOnes);
8282
8283 SDValue AllZeroes =
8284 DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0x0), dl, MVT::i32);
8285 AllZeroes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v16i8, AllZeroes);
8286
8287 // Get full vector type from predicate type
8289
8290 SDValue RecastV1;
8291 // If the real predicate is an v8i1 or v4i1 (not v16i1) then we need to recast
8292 // this to a v16i1. This cannot be done with an ordinary bitcast because the
8293 // sizes are not the same. We have to use a MVE specific PREDICATE_CAST node,
8294 // since we know in hardware the sizes are really the same.
8295 if (VT != MVT::v16i1)
8296 RecastV1 = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Pred);
8297 else
8298 RecastV1 = Pred;
8299
8300 // Select either all ones or zeroes depending upon the real predicate bits.
8301 SDValue PredAsVector =
8302 DAG.getNode(ISD::VSELECT, dl, MVT::v16i8, RecastV1, AllOnes, AllZeroes);
8303
8304 // Recast our new predicate-as-integer v16i8 vector into something
8305 // appropriate for the shuffle, i.e. v4i32 for a real v4i1 predicate.
8306 return DAG.getNode(ISD::BITCAST, dl, NewVT, PredAsVector);
8307}
8308
8310 const ARMSubtarget *ST) {
8311 EVT VT = Op.getValueType();
8313 ArrayRef<int> ShuffleMask = SVN->getMask();
8314
8315 assert(ST->hasMVEIntegerOps() &&
8316 "No support for vector shuffle of boolean predicates");
8317
8318 SDValue V1 = Op.getOperand(0);
8319 SDValue V2 = Op.getOperand(1);
8320 SDLoc dl(Op);
8321 if (isReverseMask(ShuffleMask, VT)) {
8322 SDValue cast = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, V1);
8323 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, cast);
8324 SDValue srl = DAG.getNode(ISD::SRL, dl, MVT::i32, rbit,
8325 DAG.getConstant(16, dl, MVT::i32));
8326 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, srl);
8327 }
8328
8329 // Until we can come up with optimised cases for every single vector
8330 // shuffle in existence we have chosen the least painful strategy. This is
8331 // to essentially promote the boolean predicate to a 8-bit integer, where
8332 // each predicate represents a byte. Then we fall back on a normal integer
8333 // vector shuffle and convert the result back into a predicate vector. In
8334 // many cases the generated code might be even better than scalar code
8335 // operating on bits. Just imagine trying to shuffle 8 arbitrary 2-bit
8336 // fields in a register into 8 other arbitrary 2-bit fields!
8337 SDValue PredAsVector1 = PromoteMVEPredVector(dl, V1, VT, DAG);
8338 EVT NewVT = PredAsVector1.getValueType();
8339 SDValue PredAsVector2 = V2.isUndef() ? DAG.getUNDEF(NewVT)
8340 : PromoteMVEPredVector(dl, V2, VT, DAG);
8341 assert(PredAsVector2.getValueType() == NewVT &&
8342 "Expected identical vector type in expanded i1 shuffle!");
8343
8344 // Do the shuffle!
8345 SDValue Shuffled = DAG.getVectorShuffle(NewVT, dl, PredAsVector1,
8346 PredAsVector2, ShuffleMask);
8347
8348 // Now return the result of comparing the shuffled vector with zero,
8349 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1
8350 // we convert to a v4i1 compare to fill in the two halves of the i64 as i32s.
8351 if (VT == MVT::v2i1) {
8352 SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Shuffled);
8353 SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC,
8354 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8355 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
8356 }
8357 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Shuffled,
8358 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8359}
8360
8362 ArrayRef<int> ShuffleMask,
8363 SelectionDAG &DAG) {
8364 // Attempt to lower the vector shuffle using as many whole register movs as
8365 // possible. This is useful for types smaller than 32bits, which would
8366 // often otherwise become a series for grp movs.
8367 SDLoc dl(Op);
8368 EVT VT = Op.getValueType();
8369 if (VT.getScalarSizeInBits() >= 32)
8370 return SDValue();
8371
8372 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8373 "Unexpected vector type");
8374 int NumElts = VT.getVectorNumElements();
8375 int QuarterSize = NumElts / 4;
8376 // The four final parts of the vector, as i32's
8377 SDValue Parts[4];
8378
8379 // Look for full lane vmovs like <0,1,2,3> or <u,5,6,7> etc, (but not
8380 // <u,u,u,u>), returning the vmov lane index
8381 auto getMovIdx = [](ArrayRef<int> ShuffleMask, int Start, int Length) {
8382 // Detect which mov lane this would be from the first non-undef element.
8383 int MovIdx = -1;
8384 for (int i = 0; i < Length; i++) {
8385 if (ShuffleMask[Start + i] >= 0) {
8386 if (ShuffleMask[Start + i] % Length != i)
8387 return -1;
8388 MovIdx = ShuffleMask[Start + i] / Length;
8389 break;
8390 }
8391 }
8392 // If all items are undef, leave this for other combines
8393 if (MovIdx == -1)
8394 return -1;
8395 // Check the remaining values are the correct part of the same mov
8396 for (int i = 1; i < Length; i++) {
8397 if (ShuffleMask[Start + i] >= 0 &&
8398 (ShuffleMask[Start + i] / Length != MovIdx ||
8399 ShuffleMask[Start + i] % Length != i))
8400 return -1;
8401 }
8402 return MovIdx;
8403 };
8404
8405 for (int Part = 0; Part < 4; ++Part) {
8406 // Does this part look like a mov
8407 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8408 if (Elt != -1) {
8409 SDValue Input = Op->getOperand(0);
8410 if (Elt >= 4) {
8411 Input = Op->getOperand(1);
8412 Elt -= 4;
8413 }
8414 SDValue BitCast = DAG.getBitcast(MVT::v4f32, Input);
8415 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, BitCast,
8416 DAG.getConstant(Elt, dl, MVT::i32));
8417 }
8418 }
8419
8420 // Nothing interesting found, just return
8421 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8422 return SDValue();
8423
8424 // The other parts need to be built with the old shuffle vector, cast to a
8425 // v4i32 and extract_vector_elts
8426 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8427 SmallVector<int, 16> NewShuffleMask;
8428 for (int Part = 0; Part < 4; ++Part)
8429 for (int i = 0; i < QuarterSize; i++)
8430 NewShuffleMask.push_back(
8431 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8432 SDValue NewShuffle = DAG.getVectorShuffle(
8433 VT, dl, Op->getOperand(0), Op->getOperand(1), NewShuffleMask);
8434 SDValue BitCast = DAG.getBitcast(MVT::v4f32, NewShuffle);
8435
8436 for (int Part = 0; Part < 4; ++Part)
8437 if (!Parts[Part])
8438 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32,
8439 BitCast, DAG.getConstant(Part, dl, MVT::i32));
8440 }
8441 // Build a vector out of the various parts and bitcast it back to the original
8442 // type.
8443 SDValue NewVec = DAG.getNode(ARMISD::BUILD_VECTOR, dl, MVT::v4f32, Parts);
8444 return DAG.getBitcast(VT, NewVec);
8445}
8446
8448 ArrayRef<int> ShuffleMask,
8449 SelectionDAG &DAG) {
8450 SDValue V1 = Op.getOperand(0);
8451 SDValue V2 = Op.getOperand(1);
8452 EVT VT = Op.getValueType();
8453 unsigned NumElts = VT.getVectorNumElements();
8454
8455 // An One-Off Identity mask is one that is mostly an identity mask from as
8456 // single source but contains a single element out-of-place, either from a
8457 // different vector or from another position in the same vector. As opposed to
8458 // lowering this via a ARMISD::BUILD_VECTOR we can generate an extract/insert
8459 // pair directly.
8460 auto isOneOffIdentityMask = [](ArrayRef<int> Mask, EVT VT, int BaseOffset,
8461 int &OffElement) {
8462 OffElement = -1;
8463 int NonUndef = 0;
8464 for (int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8465 if (Mask[i] == -1)
8466 continue;
8467 NonUndef++;
8468 if (Mask[i] != i + BaseOffset) {
8469 if (OffElement == -1)
8470 OffElement = i;
8471 else
8472 return false;
8473 }
8474 }
8475 return NonUndef > 2 && OffElement != -1;
8476 };
8477 int OffElement;
8478 SDValue VInput;
8479 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8480 VInput = V1;
8481 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8482 VInput = V2;
8483 else
8484 return SDValue();
8485
8486 SDLoc dl(Op);
8487 EVT SVT = VT.getScalarType() == MVT::i8 || VT.getScalarType() == MVT::i16
8488 ? MVT::i32
8489 : VT.getScalarType();
8490 SDValue Elt = DAG.getNode(
8491 ISD::EXTRACT_VECTOR_ELT, dl, SVT,
8492 ShuffleMask[OffElement] < (int)NumElts ? V1 : V2,
8493 DAG.getVectorIdxConstant(ShuffleMask[OffElement] % NumElts, dl));
8494 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, VInput, Elt,
8495 DAG.getVectorIdxConstant(OffElement % NumElts, dl));
8496}
8497
8499 const ARMSubtarget *ST) {
8500 SDValue V1 = Op.getOperand(0);
8501 SDValue V2 = Op.getOperand(1);
8502 SDLoc dl(Op);
8503 EVT VT = Op.getValueType();
8505 unsigned EltSize = VT.getScalarSizeInBits();
8506
8507 if (ST->hasMVEIntegerOps() && EltSize == 1)
8508 return LowerVECTOR_SHUFFLE_i1(Op, DAG, ST);
8509
8510 // Convert shuffles that are directly supported on NEON to target-specific
8511 // DAG nodes, instead of keeping them as shuffles and matching them again
8512 // during code selection. This is more efficient and avoids the possibility
8513 // of inconsistencies between legalization and selection.
8514 // FIXME: floating-point vectors should be canonicalized to integer vectors
8515 // of the same time so that they get CSEd properly.
8516 ArrayRef<int> ShuffleMask = SVN->getMask();
8517
8518 if (EltSize <= 32) {
8519 if (SVN->isSplat()) {
8520 int Lane = SVN->getSplatIndex();
8521 // If this is undef splat, generate it via "just" vdup, if possible.
8522 if (Lane == -1) Lane = 0;
8523
8524 // Test if V1 is a SCALAR_TO_VECTOR.
8525 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
8526 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
8527 }
8528 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR
8529 // (and probably will turn into a SCALAR_TO_VECTOR once legalization
8530 // reaches it).
8531 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR &&
8532 !isa<ConstantSDNode>(V1.getOperand(0))) {
8533 bool IsScalarToVector = true;
8534 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i)
8535 if (!V1.getOperand(i).isUndef()) {
8536 IsScalarToVector = false;
8537 break;
8538 }
8539 if (IsScalarToVector)
8540 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
8541 }
8542 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1,
8543 DAG.getConstant(Lane, dl, MVT::i32));
8544 }
8545
8546 bool ReverseVEXT = false;
8547 unsigned Imm = 0;
8548 if (ST->hasNEON() && isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8549 if (ReverseVEXT)
8550 std::swap(V1, V2);
8551 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2,
8552 DAG.getConstant(Imm, dl, MVT::i32));
8553 }
8554
8555 if (isVREVMask(ShuffleMask, VT, 64))
8556 return DAG.getNode(ARMISD::VREV64, dl, VT, V1);
8557 if (isVREVMask(ShuffleMask, VT, 32))
8558 return DAG.getNode(ARMISD::VREV32, dl, VT, V1);
8559 if (isVREVMask(ShuffleMask, VT, 16))
8560 return DAG.getNode(ARMISD::VREV16, dl, VT, V1);
8561
8562 if (ST->hasNEON() && V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) {
8563 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1,
8564 DAG.getConstant(Imm, dl, MVT::i32));
8565 }
8566
8567 // Check for Neon shuffles that modify both input vectors in place.
8568 // If both results are used, i.e., if there are two shuffles with the same
8569 // source operands and with masks corresponding to both results of one of
8570 // these operations, DAG memoization will ensure that a single node is
8571 // used for both shuffles.
8572 unsigned WhichResult = 0;
8573 bool isV_UNDEF = false;
8574 if (ST->hasNEON()) {
8575 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8576 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8577 if (isV_UNDEF)
8578 V2 = V1;
8579 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2)
8580 .getValue(WhichResult);
8581 }
8582 }
8583 if (ST->hasMVEIntegerOps()) {
8584 if (isVMOVNMask(ShuffleMask, VT, false, false))
8585 return DAG.getNode(ARMISD::VMOVN, dl, VT, V2, V1,
8586 DAG.getConstant(0, dl, MVT::i32));
8587 if (isVMOVNMask(ShuffleMask, VT, true, false))
8588 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V2,
8589 DAG.getConstant(1, dl, MVT::i32));
8590 if (isVMOVNMask(ShuffleMask, VT, true, true))
8591 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V1,
8592 DAG.getConstant(1, dl, MVT::i32));
8593 }
8594
8595 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
8596 // shuffles that produce a result larger than their operands with:
8597 // shuffle(concat(v1, undef), concat(v2, undef))
8598 // ->
8599 // shuffle(concat(v1, v2), undef)
8600 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
8601 //
8602 // This is useful in the general case, but there are special cases where
8603 // native shuffles produce larger results: the two-result ops.
8604 //
8605 // Look through the concat when lowering them:
8606 // shuffle(concat(v1, v2), undef)
8607 // ->
8608 // concat(VZIP(v1, v2):0, :1)
8609 //
8610 if (ST->hasNEON() && V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) {
8611 SDValue SubV1 = V1->getOperand(0);
8612 SDValue SubV2 = V1->getOperand(1);
8613 EVT SubVT = SubV1.getValueType();
8614
8615 // We expect these to have been canonicalized to -1.
8616 assert(llvm::all_of(ShuffleMask, [&](int i) {
8617 return i < (int)VT.getVectorNumElements();
8618 }) && "Unexpected shuffle index into UNDEF operand!");
8619
8620 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8621 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8622 if (isV_UNDEF)
8623 SubV2 = SubV1;
8624 assert((WhichResult == 0) &&
8625 "In-place shuffle of concat can only have one result!");
8626 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
8627 SubV1, SubV2);
8628 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0),
8629 Res.getValue(1));
8630 }
8631 }
8632 }
8633
8634 if (ST->hasMVEIntegerOps() && EltSize <= 32 &&
8635 (ST->hasFullFP16() || VT != MVT::v8f16)) {
8636 if (SDValue V = LowerVECTOR_SHUFFLEUsingOneOff(Op, ShuffleMask, DAG))
8637 return V;
8638
8639 for (bool Top : {false, true}) {
8640 for (bool SingleSource : {false, true}) {
8641 if (isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8642 MVT FromSVT = MVT::getIntegerVT(EltSize * 2);
8643 MVT FromVT = MVT::getVectorVT(FromSVT, ShuffleMask.size() / 2);
8644 SDValue Lo = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, FromVT, V1);
8645 SDValue Hi = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, FromVT,
8646 SingleSource ? V1 : V2);
8647 if (Top) {
8648 SDValue Amt = DAG.getConstant(EltSize, dl, FromVT);
8649 Lo = DAG.getNode(ISD::SRL, dl, FromVT, Lo, Amt);
8650 Hi = DAG.getNode(ISD::SRL, dl, FromVT, Hi, Amt);
8651 }
8652 return DAG.getNode(ARMISD::MVETRUNC, dl, VT, Lo, Hi);
8653 }
8654 }
8655 }
8656 }
8657
8658 // If the shuffle is not directly supported and it has 4 elements, use
8659 // the PerfectShuffle-generated table to synthesize it from other shuffles.
8660 unsigned NumElts = VT.getVectorNumElements();
8661 if (NumElts == 4) {
8662 unsigned PFIndexes[4];
8663 for (unsigned i = 0; i != 4; ++i) {
8664 if (ShuffleMask[i] < 0)
8665 PFIndexes[i] = 8;
8666 else
8667 PFIndexes[i] = ShuffleMask[i];
8668 }
8669
8670 // Compute the index in the perfect shuffle table.
8671 unsigned PFTableIndex =
8672 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8673 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8674 unsigned Cost = (PFEntry >> 30);
8675
8676 if (Cost <= 4) {
8677 if (ST->hasNEON())
8678 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
8679 else if (isLegalMVEShuffleOp(PFEntry)) {
8680 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8681 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8682 unsigned PFEntryLHS = PerfectShuffleTable[LHSID];
8683 unsigned PFEntryRHS = PerfectShuffleTable[RHSID];
8684 if (isLegalMVEShuffleOp(PFEntryLHS) && isLegalMVEShuffleOp(PFEntryRHS))
8685 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
8686 }
8687 }
8688 }
8689
8690 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs.
8691 if (EltSize >= 32) {
8692 // Do the expansion with floating-point types, since that is what the VFP
8693 // registers are defined to use, and since i64 is not legal.
8694 EVT EltVT = EVT::getFloatingPointVT(EltSize);
8695 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
8696 V1 = DAG.getNode(ISD::BITCAST, dl, VecVT, V1);
8697 V2 = DAG.getNode(ISD::BITCAST, dl, VecVT, V2);
8699 for (unsigned i = 0; i < NumElts; ++i) {
8700 if (ShuffleMask[i] < 0)
8701 Ops.push_back(DAG.getUNDEF(EltVT));
8702 else
8703 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
8704 ShuffleMask[i] < (int)NumElts ? V1 : V2,
8705 DAG.getConstant(ShuffleMask[i] & (NumElts-1),
8706 dl, MVT::i32)));
8707 }
8708 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
8709 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
8710 }
8711
8712 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8713 isReverseMask(ShuffleMask, VT))
8714 return LowerReverse_VECTOR_SHUFFLE(Op, DAG);
8715
8716 if (ST->hasNEON() && VT == MVT::v8i8)
8717 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
8718 return NewOp;
8719
8720 if (ST->hasMVEIntegerOps())
8721 if (SDValue NewOp = LowerVECTOR_SHUFFLEUsingMovs(Op, ShuffleMask, DAG))
8722 return NewOp;
8723
8724 // Lower v8f16 via v8i16 to avoid invalid f16 nodes.
8725 if (VT == MVT::v8f16 && !ST->hasFullFP16()) {
8726 SDValue BC0 =
8727 DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v8i16, Op.getOperand(0));
8728 SDValue BC1 =
8729 DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v8i16, Op.getOperand(1));
8730 SDValue Shuf = DAG.getVectorShuffle(MVT::v8i16, dl, BC0, BC1, ShuffleMask);
8731 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Shuf);
8732 }
8733
8734 return SDValue();
8735}
8736
8738 const ARMSubtarget *ST) {
8739 EVT VecVT = Op.getOperand(0).getValueType();
8740 SDLoc dl(Op);
8741
8742 assert(ST->hasMVEIntegerOps() &&
8743 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8744
8745 SDValue Conv =
8746 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0));
8747 unsigned Lane = Op.getConstantOperandVal(2);
8748 unsigned LaneWidth =
8750 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
8751 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32,
8752 Op.getOperand(1), DAG.getValueType(MVT::i1));
8753 SDValue BFI = DAG.getNode(ARMISD::BFI, dl, MVT::i32, Conv, Ext,
8754 DAG.getConstant(~Mask, dl, MVT::i32));
8755 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), BFI);
8756}
8757
8758SDValue ARMTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8759 SelectionDAG &DAG) const {
8760 // INSERT_VECTOR_ELT is legal only for immediate indexes.
8761 SDValue Lane = Op.getOperand(2);
8762 if (!isa<ConstantSDNode>(Lane))
8763 return SDValue();
8764
8765 SDValue Elt = Op.getOperand(1);
8766 EVT EltVT = Elt.getValueType();
8767
8768 if (Subtarget->hasMVEIntegerOps() &&
8769 Op.getValueType().getScalarSizeInBits() == 1)
8770 return LowerINSERT_VECTOR_ELT_i1(Op, DAG, Subtarget);
8771
8772 if (getTypeAction(*DAG.getContext(), EltVT) ==
8774 // INSERT_VECTOR_ELT doesn't want f16 operands promoting to f32,
8775 // but the type system will try to do that if we don't intervene.
8776 // Reinterpret any such vector-element insertion as one with the
8777 // corresponding integer types.
8778
8779 SDLoc dl(Op);
8780
8781 EVT IEltVT = MVT::getIntegerVT(EltVT.getScalarSizeInBits());
8782 assert(getTypeAction(*DAG.getContext(), IEltVT) !=
8784
8785 SDValue VecIn = Op.getOperand(0);
8786 EVT VecVT = VecIn.getValueType();
8787 EVT IVecVT = EVT::getVectorVT(*DAG.getContext(), IEltVT,
8788 VecVT.getVectorNumElements());
8789
8790 SDValue IElt = DAG.getNode(ISD::BITCAST, dl, IEltVT, Elt);
8791 SDValue IVecIn = DAG.getNode(ISD::BITCAST, dl, IVecVT, VecIn);
8792 SDValue IVecOut = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVecVT,
8793 IVecIn, IElt, Lane);
8794 return DAG.getNode(ISD::BITCAST, dl, VecVT, IVecOut);
8795 }
8796
8797 return Op;
8798}
8799
8801 const ARMSubtarget *ST) {
8802 EVT VecVT = Op.getOperand(0).getValueType();
8803 SDLoc dl(Op);
8804
8805 assert(ST->hasMVEIntegerOps() &&
8806 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8807
8808 SDValue Conv =
8809 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0));
8810 unsigned Lane = Op.getConstantOperandVal(1);
8811 unsigned LaneWidth =
8813 SDValue Shift = DAG.getNode(ISD::SRL, dl, MVT::i32, Conv,
8814 DAG.getConstant(Lane * LaneWidth, dl, MVT::i32));
8815 return Shift;
8816}
8817
8819 const ARMSubtarget *ST) {
8820 // EXTRACT_VECTOR_ELT is legal only for immediate indexes.
8821 SDValue Lane = Op.getOperand(1);
8822 if (!isa<ConstantSDNode>(Lane))
8823 return SDValue();
8824
8825 SDValue Vec = Op.getOperand(0);
8826 EVT VT = Vec.getValueType();
8827
8828 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
8829 return LowerEXTRACT_VECTOR_ELT_i1(Op, DAG, ST);
8830
8831 if (Op.getValueType() == MVT::i32 && Vec.getScalarValueSizeInBits() < 32) {
8832 SDLoc dl(Op);
8833 return DAG.getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane);
8834 }
8835
8836 return Op;
8837}
8838
8840 const ARMSubtarget *ST) {
8841 SDLoc dl(Op);
8842 assert(Op.getValueType().getScalarSizeInBits() == 1 &&
8843 "Unexpected custom CONCAT_VECTORS lowering");
8844 assert(isPowerOf2_32(Op.getNumOperands()) &&
8845 "Unexpected custom CONCAT_VECTORS lowering");
8846 assert(ST->hasMVEIntegerOps() &&
8847 "CONCAT_VECTORS lowering only supported for MVE");
8848
8849 auto ConcatPair = [&](SDValue V1, SDValue V2) {
8850 EVT Op1VT = V1.getValueType();
8851 EVT Op2VT = V2.getValueType();
8852 assert(Op1VT == Op2VT && "Operand types don't match!");
8853 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
8854 "Unexpected i1 concat operations!");
8855 EVT VT = Op1VT.getDoubleNumVectorElementsVT(*DAG.getContext());
8856
8857 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
8858 SDValue NewV2 = PromoteMVEPredVector(dl, V2, Op2VT, DAG);
8859
8860 // We now have Op1 + Op2 promoted to vectors of integers, where v8i1 gets
8861 // promoted to v8i16, etc.
8862 MVT ElType =
8864 unsigned NumElts = 2 * Op1VT.getVectorNumElements();
8865
8866 EVT ConcatVT = MVT::getVectorVT(ElType, NumElts);
8867 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
8868 // Use MVETRUNC to truncate the combined NewV1::NewV2 into the smaller
8869 // ConcatVT.
8870 SDValue ConVec =
8871 DAG.getNode(ARMISD::MVETRUNC, dl, ConcatVT, NewV1, NewV2);
8872 return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec,
8873 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8874 }
8875
8876 // Extract the vector elements from Op1 and Op2 one by one and truncate them
8877 // to be the right size for the destination. For example, if Op1 is v4i1
8878 // then the promoted vector is v4i32. The result of concatenation gives a
8879 // v8i1, which when promoted is v8i16. That means each i32 element from Op1
8880 // needs truncating to i16 and inserting in the result.
8881 auto ExtractInto = [&DAG, &dl](SDValue NewV, SDValue ConVec, unsigned &j) {
8882 EVT NewVT = NewV.getValueType();
8883 EVT ConcatVT = ConVec.getValueType();
8884 unsigned ExtScale = 1;
8885 if (NewVT == MVT::v2f64) {
8886 NewV = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, NewV);
8887 ExtScale = 2;
8888 }
8889 for (unsigned i = 0, e = NewVT.getVectorNumElements(); i < e; i++, j++) {
8890 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV,
8891 DAG.getIntPtrConstant(i * ExtScale, dl));
8892 ConVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ConcatVT, ConVec, Elt,
8893 DAG.getConstant(j, dl, MVT::i32));
8894 }
8895 return ConVec;
8896 };
8897 unsigned j = 0;
8898 SDValue ConVec = DAG.getNode(ISD::UNDEF, dl, ConcatVT);
8899 ConVec = ExtractInto(NewV1, ConVec, j);
8900 ConVec = ExtractInto(NewV2, ConVec, j);
8901
8902 // Now return the result of comparing the subvector with zero, which will
8903 // generate a real predicate, i.e. v4i1, v8i1 or v16i1.
8904 return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec,
8905 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8906 };
8907
8908 // Concat each pair of subvectors and pack into the lower half of the array.
8909 SmallVector<SDValue> ConcatOps(Op->ops());
8910 while (ConcatOps.size() > 1) {
8911 for (unsigned I = 0, E = ConcatOps.size(); I != E; I += 2) {
8912 SDValue V1 = ConcatOps[I];
8913 SDValue V2 = ConcatOps[I + 1];
8914 ConcatOps[I / 2] = ConcatPair(V1, V2);
8915 }
8916 ConcatOps.resize(ConcatOps.size() / 2);
8917 }
8918 return ConcatOps[0];
8919}
8920
8922 const ARMSubtarget *ST) {
8923 EVT VT = Op->getValueType(0);
8924 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
8925 return LowerCONCAT_VECTORS_i1(Op, DAG, ST);
8926
8927 // The only time a CONCAT_VECTORS operation can have legal types is when
8928 // two 64-bit vectors are concatenated to a 128-bit vector.
8929 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 &&
8930 "unexpected CONCAT_VECTORS");
8931 SDLoc dl(Op);
8932 SDValue Val = DAG.getUNDEF(MVT::v2f64);
8933 SDValue Op0 = Op.getOperand(0);
8934 SDValue Op1 = Op.getOperand(1);
8935 if (!Op0.isUndef())
8936 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
8937 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0),
8938 DAG.getIntPtrConstant(0, dl));
8939 if (!Op1.isUndef())
8940 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
8941 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1),
8942 DAG.getIntPtrConstant(1, dl));
8943 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val);
8944}
8945
8947 const ARMSubtarget *ST) {
8948 SDValue V1 = Op.getOperand(0);
8949 SDValue V2 = Op.getOperand(1);
8950 SDLoc dl(Op);
8951 EVT VT = Op.getValueType();
8952 EVT Op1VT = V1.getValueType();
8953 unsigned NumElts = VT.getVectorNumElements();
8954 unsigned Index = V2->getAsZExtVal();
8955
8956 assert(VT.getScalarSizeInBits() == 1 &&
8957 "Unexpected custom EXTRACT_SUBVECTOR lowering");
8958 assert(ST->hasMVEIntegerOps() &&
8959 "EXTRACT_SUBVECTOR lowering only supported for MVE");
8960
8961 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
8962
8963 // We now have Op1 promoted to a vector of integers, where v8i1 gets
8964 // promoted to v8i16, etc.
8965
8967
8968 if (NumElts == 2) {
8969 EVT SubVT = MVT::v4i32;
8970 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
8971 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) {
8972 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
8973 DAG.getIntPtrConstant(i, dl));
8974 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
8975 DAG.getConstant(j, dl, MVT::i32));
8976 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
8977 DAG.getConstant(j + 1, dl, MVT::i32));
8978 }
8979 SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, SubVec,
8980 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8981 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
8982 }
8983
8984 EVT SubVT = MVT::getVectorVT(ElType, NumElts);
8985 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
8986 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) {
8987 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
8988 DAG.getIntPtrConstant(i, dl));
8989 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
8990 DAG.getConstant(j, dl, MVT::i32));
8991 }
8992
8993 // Now return the result of comparing the subvector with zero,
8994 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
8995 return DAG.getNode(ARMISD::VCMPZ, dl, VT, SubVec,
8996 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8997}
8998
8999// Turn a truncate into a predicate (an i1 vector) into icmp(and(x, 1), 0).
9001 const ARMSubtarget *ST) {
9002 assert(ST->hasMVEIntegerOps() && "Expected MVE!");
9003 EVT VT = N->getValueType(0);
9004 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9005 "Expected a vector i1 type!");
9006 SDValue Op = N->getOperand(0);
9007 EVT FromVT = Op.getValueType();
9008 SDLoc DL(N);
9009
9010 SDValue And =
9011 DAG.getNode(ISD::AND, DL, FromVT, Op, DAG.getConstant(1, DL, FromVT));
9012 return DAG.getNode(ISD::SETCC, DL, VT, And, DAG.getConstant(0, DL, FromVT),
9013 DAG.getCondCode(ISD::SETNE));
9014}
9015
9017 const ARMSubtarget *Subtarget) {
9018 if (!Subtarget->hasMVEIntegerOps())
9019 return SDValue();
9020
9021 EVT ToVT = N->getValueType(0);
9022 if (ToVT.getScalarType() == MVT::i1)
9023 return LowerTruncatei1(N, DAG, Subtarget);
9024
9025 // MVE does not have a single instruction to perform the truncation of a v4i32
9026 // into the lower half of a v8i16, in the same way that a NEON vmovn would.
9027 // Most of the instructions in MVE follow the 'Beats' system, where moving
9028 // values from different lanes is usually something that the instructions
9029 // avoid.
9030 //
9031 // Instead it has top/bottom instructions such as VMOVLT/B and VMOVNT/B,
9032 // which take a the top/bottom half of a larger lane and extend it (or do the
9033 // opposite, truncating into the top/bottom lane from a larger lane). Note
9034 // that because of the way we widen lanes, a v4i16 is really a v4i32 using the
9035 // bottom 16bits from each vector lane. This works really well with T/B
9036 // instructions, but that doesn't extend to v8i32->v8i16 where the lanes need
9037 // to move order.
9038 //
9039 // But truncates and sext/zext are always going to be fairly common from llvm.
9040 // We have several options for how to deal with them:
9041 // - Wherever possible combine them into an instruction that makes them
9042 // "free". This includes loads/stores, which can perform the trunc as part
9043 // of the memory operation. Or certain shuffles that can be turned into
9044 // VMOVN/VMOVL.
9045 // - Lane Interleaving to transform blocks surrounded by ext/trunc. So
9046 // trunc(mul(sext(a), sext(b))) may become
9047 // VMOVNT(VMUL(VMOVLB(a), VMOVLB(b)), VMUL(VMOVLT(a), VMOVLT(b))). (Which in
9048 // this case can use VMULL). This is performed in the
9049 // MVELaneInterleavingPass.
9050 // - Otherwise we have an option. By default we would expand the
9051 // zext/sext/trunc into a series of lane extract/inserts going via GPR
9052 // registers. One for each vector lane in the vector. This can obviously be
9053 // very expensive.
9054 // - The other option is to use the fact that loads/store can extend/truncate
9055 // to turn a trunc into two truncating stack stores and a stack reload. This
9056 // becomes 3 back-to-back memory operations, but at least that is less than
9057 // all the insert/extracts.
9058 //
9059 // In order to do the last, we convert certain trunc's into MVETRUNC, which
9060 // are either optimized where they can be, or eventually lowered into stack
9061 // stores/loads. This prevents us from splitting a v8i16 trunc into two stores
9062 // two early, where other instructions would be better, and stops us from
9063 // having to reconstruct multiple buildvector shuffles into loads/stores.
9064 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9065 return SDValue();
9066 EVT FromVT = N->getOperand(0).getValueType();
9067 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9068 return SDValue();
9069
9070 SDValue Lo, Hi;
9071 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
9072 SDLoc DL(N);
9073 return DAG.getNode(ARMISD::MVETRUNC, DL, ToVT, Lo, Hi);
9074}
9075
9077 const ARMSubtarget *Subtarget) {
9078 if (!Subtarget->hasMVEIntegerOps())
9079 return SDValue();
9080
9081 // See LowerTruncate above for an explanation of MVEEXT/MVETRUNC.
9082
9083 EVT ToVT = N->getValueType(0);
9084 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9085 return SDValue();
9086 SDValue Op = N->getOperand(0);
9087 EVT FromVT = Op.getValueType();
9088 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9089 return SDValue();
9090
9091 SDLoc DL(N);
9092 EVT ExtVT = ToVT.getHalfNumVectorElementsVT(*DAG.getContext());
9093 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8)
9094 ExtVT = MVT::v8i16;
9095
9096 unsigned Opcode =
9098 SDValue Ext = DAG.getNode(Opcode, DL, DAG.getVTList(ExtVT, ExtVT), Op);
9099 SDValue Ext1 = Ext.getValue(1);
9100
9101 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8) {
9102 Ext = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext);
9103 Ext1 = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext1);
9104 }
9105
9106 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Ext, Ext1);
9107}
9108
9109/// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
9110/// element has been zero/sign-extended, depending on the isSigned parameter,
9111/// from an integer type half its size.
9113 bool isSigned) {
9114 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32.
9115 EVT VT = N->getValueType(0);
9116 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) {
9117 SDNode *BVN = N->getOperand(0).getNode();
9118 if (BVN->getValueType(0) != MVT::v4i32 ||
9119 BVN->getOpcode() != ISD::BUILD_VECTOR)
9120 return false;
9121 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9122 unsigned HiElt = 1 - LoElt;
9127 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9128 return false;
9129 if (isSigned) {
9130 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 &&
9131 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32)
9132 return true;
9133 } else {
9134 if (Hi0->isZero() && Hi1->isZero())
9135 return true;
9136 }
9137 return false;
9138 }
9139
9140 if (N->getOpcode() != ISD::BUILD_VECTOR)
9141 return false;
9142
9143 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
9144 SDNode *Elt = N->getOperand(i).getNode();
9146 unsigned EltSize = VT.getScalarSizeInBits();
9147 unsigned HalfSize = EltSize / 2;
9148 if (isSigned) {
9149 if (!isIntN(HalfSize, C->getSExtValue()))
9150 return false;
9151 } else {
9152 if (!isUIntN(HalfSize, C->getZExtValue()))
9153 return false;
9154 }
9155 continue;
9156 }
9157 return false;
9158 }
9159
9160 return true;
9161}
9162
9163/// isSignExtended - Check if a node is a vector value that is sign-extended
9164/// or a constant BUILD_VECTOR with sign-extended elements.
9166 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N))
9167 return true;
9168 if (isExtendedBUILD_VECTOR(N, DAG, true))
9169 return true;
9170 return false;
9171}
9172
9173/// isZeroExtended - Check if a node is a vector value that is zero-extended (or
9174/// any-extended) or a constant BUILD_VECTOR with zero-extended elements.
9176 if (N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND ||
9178 return true;
9179 if (isExtendedBUILD_VECTOR(N, DAG, false))
9180 return true;
9181 return false;
9182}
9183
9184static EVT getExtensionTo64Bits(const EVT &OrigVT) {
9185 if (OrigVT.getSizeInBits() >= 64)
9186 return OrigVT;
9187
9188 assert(OrigVT.isSimple() && "Expecting a simple value type");
9189
9190 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
9191 switch (OrigSimpleTy) {
9192 default: llvm_unreachable("Unexpected Vector Type");
9193 case MVT::v2i8:
9194 case MVT::v2i16:
9195 return MVT::v2i32;
9196 case MVT::v4i8:
9197 return MVT::v4i16;
9198 }
9199}
9200
9201/// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
9202/// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
9203/// We insert the required extension here to get the vector to fill a D register.
9205 const EVT &OrigTy,
9206 const EVT &ExtTy,
9207 unsigned ExtOpcode) {
9208 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
9209 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
9210 // 64-bits we need to insert a new extension so that it will be 64-bits.
9211 assert(ExtTy.is128BitVector() && "Unexpected extension size");
9212 if (OrigTy.getSizeInBits() >= 64)
9213 return N;
9214
9215 // Must extend size to at least 64 bits to be used as an operand for VMULL.
9216 EVT NewVT = getExtensionTo64Bits(OrigTy);
9217
9218 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
9219}
9220
9221/// SkipLoadExtensionForVMULL - return a load of the original vector size that
9222/// does not do any sign/zero extension. If the original vector is less
9223/// than 64 bits, an appropriate extension will be added after the load to
9224/// reach a total size of 64 bits. We have to add the extension separately
9225/// because ARM does not have a sign/zero extending load for vectors.
9227 EVT ExtendedTy = getExtensionTo64Bits(LD->getMemoryVT());
9228
9229 // The load already has the right type.
9230 if (ExtendedTy == LD->getMemoryVT())
9231 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(),
9232 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9233 LD->getMemOperand()->getFlags());
9234
9235 // We need to create a zextload/sextload. We cannot just create a load
9236 // followed by a zext/zext node because LowerMUL is also run during normal
9237 // operation legalization where we can't create illegal types.
9238 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy,
9239 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9240 LD->getMemoryVT(), LD->getAlign(),
9241 LD->getMemOperand()->getFlags());
9242}
9243
9244/// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
9245/// ANY_EXTEND, extending load, or BUILD_VECTOR with extended elements, return
9246/// the unextended value. The unextended vector should be 64 bits so that it can
9247/// be used as an operand to a VMULL instruction. If the original vector size
9248/// before extension is less than 64 bits we add a an extension to resize
9249/// the vector to 64 bits.
9251 if (N->getOpcode() == ISD::SIGN_EXTEND ||
9252 N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND)
9253 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG,
9254 N->getOperand(0)->getValueType(0),
9255 N->getValueType(0),
9256 N->getOpcode());
9257
9258 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
9259 assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) &&
9260 "Expected extending load");
9261
9262 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG);
9263 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), newLoad.getValue(1));
9264 unsigned Opcode = ISD::isSEXTLoad(LD) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
9265 SDValue extLoad =
9266 DAG.getNode(Opcode, SDLoc(newLoad), LD->getValueType(0), newLoad);
9267 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 0), extLoad);
9268
9269 return newLoad;
9270 }
9271
9272 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will
9273 // have been legalized as a BITCAST from v4i32.
9274 if (N->getOpcode() == ISD::BITCAST) {
9275 SDNode *BVN = N->getOperand(0).getNode();
9277 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR");
9278 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9279 return DAG.getBuildVector(
9280 MVT::v2i32, SDLoc(N),
9281 {BVN->getOperand(LowElt), BVN->getOperand(LowElt + 2)});
9282 }
9283 // Construct a new BUILD_VECTOR with elements truncated to half the size.
9284 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
9285 EVT VT = N->getValueType(0);
9286 unsigned EltSize = VT.getScalarSizeInBits() / 2;
9287 unsigned NumElts = VT.getVectorNumElements();
9288 MVT TruncVT = MVT::getIntegerVT(EltSize);
9290 SDLoc dl(N);
9291 for (unsigned i = 0; i != NumElts; ++i) {
9292 const APInt &CInt = N->getConstantOperandAPInt(i);
9293 // Element types smaller than 32 bits are not legal, so use i32 elements.
9294 // The values are implicitly truncated so sext vs. zext doesn't matter.
9295 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
9296 }
9297 return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
9298}
9299
9300static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
9301 unsigned Opcode = N->getOpcode();
9302 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9303 SDNode *N0 = N->getOperand(0).getNode();
9304 SDNode *N1 = N->getOperand(1).getNode();
9305 return N0->hasOneUse() && N1->hasOneUse() &&
9306 isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
9307 }
9308 return false;
9309}
9310
9311static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
9312 unsigned Opcode = N->getOpcode();
9313 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9314 SDNode *N0 = N->getOperand(0).getNode();
9315 SDNode *N1 = N->getOperand(1).getNode();
9316 return N0->hasOneUse() && N1->hasOneUse() &&
9317 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
9318 }
9319 return false;
9320}
9321
9323 // Multiplications are only custom-lowered for 128-bit vectors so that
9324 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
9325 EVT VT = Op.getValueType();
9326 assert(VT.is128BitVector() && VT.isInteger() &&
9327 "unexpected type for custom-lowering ISD::MUL");
9328 SDNode *N0 = Op.getOperand(0).getNode();
9329 SDNode *N1 = Op.getOperand(1).getNode();
9330 unsigned NewOpc = 0;
9331 bool isMLA = false;
9332 bool isN0SExt = isSignExtended(N0, DAG);
9333 bool isN1SExt = isSignExtended(N1, DAG);
9334 if (isN0SExt && isN1SExt)
9335 NewOpc = ARMISD::VMULLs;
9336 else {
9337 bool isN0ZExt = isZeroExtended(N0, DAG);
9338 bool isN1ZExt = isZeroExtended(N1, DAG);
9339 if (isN0ZExt && isN1ZExt)
9340 NewOpc = ARMISD::VMULLu;
9341 else if (isN1SExt || isN1ZExt) {
9342 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
9343 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
9344 if (isN1SExt && isAddSubSExt(N0, DAG)) {
9345 NewOpc = ARMISD::VMULLs;
9346 isMLA = true;
9347 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
9348 NewOpc = ARMISD::VMULLu;
9349 isMLA = true;
9350 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
9351 std::swap(N0, N1);
9352 NewOpc = ARMISD::VMULLu;
9353 isMLA = true;
9354 }
9355 }
9356
9357 if (!NewOpc) {
9358 if (VT == MVT::v2i64)
9359 // Fall through to expand this. It is not legal.
9360 return SDValue();
9361 else
9362 // Other vector multiplications are legal.
9363 return Op;
9364 }
9365 }
9366
9367 // Legalize to a VMULL instruction.
9368 SDLoc DL(Op);
9369 SDValue Op0;
9370 SDValue Op1 = SkipExtensionForVMULL(N1, DAG);
9371 if (!isMLA) {
9372 Op0 = SkipExtensionForVMULL(N0, DAG);
9374 Op1.getValueType().is64BitVector() &&
9375 "unexpected types for extended operands to VMULL");
9376 return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
9377 }
9378
9379 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during
9380 // isel lowering to take advantage of no-stall back to back vmul + vmla.
9381 // vmull q0, d4, d6
9382 // vmlal q0, d5, d6
9383 // is faster than
9384 // vaddl q0, d4, d5
9385 // vmovl q1, d6
9386 // vmul q0, q0, q1
9387 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG);
9388 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG);
9389 EVT Op1VT = Op1.getValueType();
9390 return DAG.getNode(N0->getOpcode(), DL, VT,
9391 DAG.getNode(NewOpc, DL, VT,
9392 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
9393 DAG.getNode(NewOpc, DL, VT,
9394 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
9395}
9396
9398 SelectionDAG &DAG) {
9399 // TODO: Should this propagate fast-math-flags?
9400
9401 // Convert to float
9402 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo));
9403 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo));
9404 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, X);
9405 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Y);
9406 X = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, X);
9407 Y = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, Y);
9408 // Get reciprocal estimate.
9409 // float4 recip = vrecpeq_f32(yf);
9410 Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9411 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9412 Y);
9413 // Because char has a smaller range than uchar, we can actually get away
9414 // without any newton steps. This requires that we use a weird bias
9415 // of 0xb000, however (again, this has been exhaustively tested).
9416 // float4 result = as_float4(as_int4(xf*recip) + 0xb000);
9417 X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y);
9418 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X);
9419 Y = DAG.getConstant(0xb000, dl, MVT::v4i32);
9420 X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y);
9421 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X);
9422 // Convert back to short.
9423 X = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, X);
9424 X = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, X);
9425 return X;
9426}
9427
9429 SelectionDAG &DAG) {
9430 // TODO: Should this propagate fast-math-flags?
9431
9432 SDValue N2;
9433 // Convert to float.
9434 // float4 yf = vcvt_f32_s32(vmovl_s16(y));
9435 // float4 xf = vcvt_f32_s32(vmovl_s16(x));
9436 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N0);
9437 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N1);
9438 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
9439 N1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
9440
9441 // Use reciprocal estimate and one refinement step.
9442 // float4 recip = vrecpeq_f32(yf);
9443 // recip *= vrecpsq_f32(yf, recip);
9444 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9445 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9446 N1);
9447 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9448 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9449 N1, N2);
9450 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9451 // Because short has a smaller range than ushort, we can actually get away
9452 // with only a single newton step. This requires that we use a weird bias
9453 // of 89, however (again, this has been exhaustively tested).
9454 // float4 result = as_float4(as_int4(xf*recip) + 0x89);
9455 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
9456 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9457 N1 = DAG.getConstant(0x89, dl, MVT::v4i32);
9458 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
9459 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9460 // Convert back to integer and return.
9461 // return vmovn_s32(vcvt_s32_f32(result));
9462 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
9463 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
9464 return N0;
9465}
9466
9468 const ARMSubtarget *ST) {
9469 EVT VT = Op.getValueType();
9470 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9471 "unexpected type for custom-lowering ISD::SDIV");
9472
9473 SDLoc dl(Op);
9474 SDValue N0 = Op.getOperand(0);
9475 SDValue N1 = Op.getOperand(1);
9476 SDValue N2, N3;
9477
9478 if (VT == MVT::v8i8) {
9479 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N0);
9480 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1);
9481
9482 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9483 DAG.getIntPtrConstant(4, dl));
9484 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9485 DAG.getIntPtrConstant(4, dl));
9486 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9487 DAG.getIntPtrConstant(0, dl));
9488 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9489 DAG.getIntPtrConstant(0, dl));
9490
9491 N0 = LowerSDIV_v4i8(N0, N1, dl, DAG); // v4i16
9492 N2 = LowerSDIV_v4i8(N2, N3, dl, DAG); // v4i16
9493
9494 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
9495 N0 = LowerCONCAT_VECTORS(N0, DAG, ST);
9496
9497 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i8, N0);
9498 return N0;
9499 }
9500 return LowerSDIV_v4i16(N0, N1, dl, DAG);
9501}
9502
9504 const ARMSubtarget *ST) {
9505 // TODO: Should this propagate fast-math-flags?
9506 EVT VT = Op.getValueType();
9507 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9508 "unexpected type for custom-lowering ISD::UDIV");
9509
9510 SDLoc dl(Op);
9511 SDValue N0 = Op.getOperand(0);
9512 SDValue N1 = Op.getOperand(1);
9513 SDValue N2, N3;
9514
9515 if (VT == MVT::v8i8) {
9516 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N0);
9517 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1);
9518
9519 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9520 DAG.getIntPtrConstant(4, dl));
9521 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9522 DAG.getIntPtrConstant(4, dl));
9523 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9524 DAG.getIntPtrConstant(0, dl));
9525 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9526 DAG.getIntPtrConstant(0, dl));
9527
9528 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
9529 N2 = LowerSDIV_v4i16(N2, N3, dl, DAG); // v4i16
9530
9531 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
9532 N0 = LowerCONCAT_VECTORS(N0, DAG, ST);
9533
9534 N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8,
9535 DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, dl,
9536 MVT::i32),
9537 N0);
9538 return N0;
9539 }
9540
9541 // v4i16 sdiv ... Convert to float.
9542 // float4 yf = vcvt_f32_s32(vmovl_u16(y));
9543 // float4 xf = vcvt_f32_s32(vmovl_u16(x));
9544 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N0);
9545 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N1);
9546 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
9547 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
9548
9549 // Use reciprocal estimate and two refinement steps.
9550 // float4 recip = vrecpeq_f32(yf);
9551 // recip *= vrecpsq_f32(yf, recip);
9552 // recip *= vrecpsq_f32(yf, recip);
9553 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9554 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9555 BN1);
9556 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9557 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9558 BN1, N2);
9559 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9560 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9561 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9562 BN1, N2);
9563 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9564 // Simply multiplying by the reciprocal estimate can leave us a few ulps
9565 // too low, so we add 2 ulps (exhaustive testing shows that this is enough,
9566 // and that it will never cause us to return an answer too large).
9567 // float4 result = as_float4(as_int4(xf*recip) + 2);
9568 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
9569 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9570 N1 = DAG.getConstant(2, dl, MVT::v4i32);
9571 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
9572 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9573 // Convert back to integer and return.
9574 // return vmovn_u32(vcvt_s32_f32(result));
9575 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
9576 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
9577 return N0;
9578}
9579
9581 unsigned Opcode, bool IsSigned) {
9582 EVT VT0 = Op.getValue(0).getValueType();
9583 EVT VT1 = Op.getValue(1).getValueType();
9584
9585 bool InvertCarry = Opcode == ARMISD::SUBE;
9586 SDValue OpLHS = Op.getOperand(0);
9587 SDValue OpRHS = Op.getOperand(1);
9588 SDValue OpCarryIn = valueToCarryFlag(Op.getOperand(2), DAG, InvertCarry);
9589
9590 SDLoc DL(Op);
9591
9592 SDValue Result = DAG.getNode(Opcode, DL, DAG.getVTList(VT0, MVT::i32), OpLHS,
9593 OpRHS, OpCarryIn);
9594
9595 SDValue OutFlag =
9596 IsSigned ? overflowFlagToValue(Result.getValue(1), VT1, DAG)
9597 : carryFlagToValue(Result.getValue(1), VT1, DAG, InvertCarry);
9598
9599 return DAG.getMergeValues({Result, OutFlag}, DL);
9600}
9601
9602SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
9603 bool Signed,
9604 SDValue &Chain) const {
9605 EVT VT = Op.getValueType();
9606 assert((VT == MVT::i32 || VT == MVT::i64) &&
9607 "unexpected type for custom lowering DIV");
9608 SDLoc dl(Op);
9609
9610 const auto &DL = DAG.getDataLayout();
9611 RTLIB::Libcall LC;
9612 if (Signed)
9613 LC = VT == MVT::i32 ? RTLIB::SDIVREM_I32 : RTLIB::SDIVREM_I64;
9614 else
9615 LC = VT == MVT::i32 ? RTLIB::UDIVREM_I32 : RTLIB::UDIVREM_I64;
9616
9617 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
9618 SDValue ES = DAG.getExternalSymbol(LCImpl, getPointerTy(DL));
9619
9621
9622 for (auto AI : {1, 0}) {
9623 SDValue Operand = Op.getOperand(AI);
9624 Args.emplace_back(Operand,
9625 Operand.getValueType().getTypeForEVT(*DAG.getContext()));
9626 }
9627
9628 CallLoweringInfo CLI(DAG);
9629 CLI.setDebugLoc(dl).setChain(Chain).setCallee(
9631 VT.getTypeForEVT(*DAG.getContext()), ES, std::move(Args));
9632
9633 return LowerCallTo(CLI).first;
9634}
9635
9636// This is a code size optimisation: return the original SDIV node to
9637// DAGCombiner when we don't want to expand SDIV into a sequence of
9638// instructions, and an empty node otherwise which will cause the
9639// SDIV to be expanded in DAGCombine.
9640SDValue
9641ARMTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
9642 SelectionDAG &DAG,
9643 SmallVectorImpl<SDNode *> &Created) const {
9644 // TODO: Support SREM
9645 if (N->getOpcode() != ISD::SDIV)
9646 return SDValue();
9647
9648 const auto &ST = DAG.getSubtarget<ARMSubtarget>();
9649 const bool MinSize = ST.hasMinSize();
9650 const bool HasDivide = ST.isThumb() ? ST.hasDivideInThumbMode()
9651 : ST.hasDivideInARMMode();
9652
9653 // Don't touch vector types; rewriting this may lead to scalarizing
9654 // the int divs.
9655 if (N->getOperand(0).getValueType().isVector())
9656 return SDValue();
9657
9658 // Bail if MinSize is not set, and also for both ARM and Thumb mode we need
9659 // hwdiv support for this to be really profitable.
9660 if (!(MinSize && HasDivide))
9661 return SDValue();
9662
9663 // ARM mode is a bit simpler than Thumb: we can handle large power
9664 // of 2 immediates with 1 mov instruction; no further checks required,
9665 // just return the sdiv node.
9666 if (!ST.isThumb())
9667 return SDValue(N, 0);
9668
9669 // In Thumb mode, immediates larger than 128 need a wide 4-byte MOV,
9670 // and thus lose the code size benefits of a MOVS that requires only 2.
9671 // TargetTransformInfo and 'getIntImmCodeSizeCost' could be helpful here,
9672 // but as it's doing exactly this, it's not worth the trouble to get TTI.
9673 if (Divisor.sgt(128))
9674 return SDValue();
9675
9676 return SDValue(N, 0);
9677}
9678
9679SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
9680 bool Signed) const {
9681 assert(Op.getValueType() == MVT::i32 &&
9682 "unexpected type for custom lowering DIV");
9683 SDLoc dl(Op);
9684
9685 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other,
9686 DAG.getEntryNode(), Op.getOperand(1));
9687
9688 return LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
9689}
9690
9692 SDLoc DL(N);
9693 SDValue Op = N->getOperand(1);
9694 if (N->getValueType(0) == MVT::i32)
9695 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain, Op);
9696 SDValue Lo, Hi;
9697 std::tie(Lo, Hi) = DAG.SplitScalar(Op, DL, MVT::i32, MVT::i32);
9698 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain,
9699 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi));
9700}
9701
9702void ARMTargetLowering::ExpandDIV_Windows(
9703 SDValue Op, SelectionDAG &DAG, bool Signed,
9705 const auto &DL = DAG.getDataLayout();
9706
9707 assert(Op.getValueType() == MVT::i64 &&
9708 "unexpected type for custom lowering DIV");
9709 SDLoc dl(Op);
9710
9711 SDValue DBZCHK = WinDBZCheckDenominator(DAG, Op.getNode(), DAG.getEntryNode());
9712
9713 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
9714
9715 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result);
9716 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result,
9717 DAG.getConstant(32, dl, getPointerTy(DL)));
9718 Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper);
9719
9720 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lower, Upper));
9721}
9722
9723std::pair<SDValue, SDValue>
9724ARMTargetLowering::LowerAEABIUnalignedLoad(SDValue Op,
9725 SelectionDAG &DAG) const {
9726 // If we have an unaligned load from a i32 or i64 that would normally be
9727 // split into separate ldrb's, we can use the __aeabi_uread4/__aeabi_uread8
9728 // functions instead.
9729 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
9730 EVT MemVT = LD->getMemoryVT();
9731 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9732 return std::make_pair(SDValue(), SDValue());
9733
9734 const auto &MF = DAG.getMachineFunction();
9735 unsigned AS = LD->getAddressSpace();
9736 Align Alignment = LD->getAlign();
9737 const DataLayout &DL = DAG.getDataLayout();
9738 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9739 RTLIB::Libcall LC =
9740 (MemVT == MVT::i32) ? RTLIB::AEABI_UREAD4 : RTLIB::AEABI_UREAD8;
9741
9742 if (MF.getFunction().hasMinSize() && !AllowsUnaligned &&
9743 Alignment <= llvm::Align(2) && DAG.getLibcalls().getLibcallImpl(LC)) {
9744 MakeLibCallOptions Opts;
9745 SDLoc dl(Op);
9746
9747 auto Pair = makeLibCall(DAG, LC, MemVT.getSimpleVT(), LD->getBasePtr(),
9748 Opts, dl, LD->getChain());
9749
9750 // If necessary, extend the node to 64bit
9751 if (LD->getExtensionType() != ISD::NON_EXTLOAD) {
9752 unsigned ExtType = LD->getExtensionType() == ISD::SEXTLOAD
9755 SDValue EN = DAG.getNode(ExtType, dl, LD->getValueType(0), Pair.first);
9756 Pair.first = EN;
9757 }
9758 return Pair;
9759 }
9760
9761 // Default expand to individual loads
9762 if (!allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Alignment))
9763 return expandUnalignedLoad(LD, DAG);
9764 return std::make_pair(SDValue(), SDValue());
9765}
9766
9767SDValue ARMTargetLowering::LowerAEABIUnalignedStore(SDValue Op,
9768 SelectionDAG &DAG) const {
9769 // If we have an unaligned store to a i32 or i64 that would normally be
9770 // split into separate ldrb's, we can use the __aeabi_uwrite4/__aeabi_uwrite8
9771 // functions instead.
9772 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
9773 EVT MemVT = ST->getMemoryVT();
9774 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9775 return SDValue();
9776
9777 const auto &MF = DAG.getMachineFunction();
9778 unsigned AS = ST->getAddressSpace();
9779 Align Alignment = ST->getAlign();
9780 const DataLayout &DL = DAG.getDataLayout();
9781 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9782 RTLIB::Libcall LC =
9783 (MemVT == MVT::i32) ? RTLIB::AEABI_UWRITE4 : RTLIB::AEABI_UWRITE8;
9784
9785 if (MF.getFunction().hasMinSize() && !AllowsUnaligned &&
9786 Alignment <= llvm::Align(2) && DAG.getLibcalls().getLibcallImpl(LC)) {
9787
9788 SDLoc dl(Op);
9789
9790 // If necessary, trunc the value to 32bit
9791 SDValue StoreVal = ST->getOperand(1);
9792 if (ST->isTruncatingStore())
9793 StoreVal = DAG.getNode(ISD::TRUNCATE, dl, MemVT, ST->getOperand(1));
9794
9795 MakeLibCallOptions Opts;
9796 auto CallResult =
9797 makeLibCall(DAG, LC, MVT::isVoid, {StoreVal, ST->getBasePtr()}, Opts,
9798 dl, ST->getChain());
9799
9800 return CallResult.second;
9801 }
9802
9803 // Default expand to individual stores
9804 if (!allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Alignment))
9805 return expandUnalignedStore(ST, DAG);
9806 return SDValue();
9807}
9808
9810 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
9811 EVT MemVT = LD->getMemoryVT();
9812 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9813 MemVT == MVT::v16i1) &&
9814 "Expected a predicate type!");
9815 assert(MemVT == Op.getValueType());
9816 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
9817 "Expected a non-extending load");
9818 assert(LD->isUnindexed() && "Expected a unindexed load");
9819
9820 // The basic MVE VLDR on a v2i1/v4i1/v8i1 actually loads the entire 16bit
9821 // predicate, with the "v4i1" bits spread out over the 16 bits loaded. We
9822 // need to make sure that 8/4/2 bits are actually loaded into the correct
9823 // place, which means loading the value and then shuffling the values into
9824 // the bottom bits of the predicate.
9825 // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect
9826 // for BE).
9827 // Speaking of BE, apparently the rest of llvm will assume a reverse order to
9828 // a natural VMSR(load), so needs to be reversed.
9829
9830 SDLoc dl(Op);
9831 SDValue Load = DAG.getExtLoad(
9832 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
9834 LD->getMemOperand());
9835 SDValue Val = Load;
9836 if (DAG.getDataLayout().isBigEndian())
9837 Val = DAG.getNode(ISD::SRL, dl, MVT::i32,
9838 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, Load),
9839 DAG.getConstant(32 - MemVT.getSizeInBits(), dl, MVT::i32));
9840 SDValue Pred = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Val);
9841 if (MemVT != MVT::v16i1)
9842 Pred = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MemVT, Pred,
9843 DAG.getConstant(0, dl, MVT::i32));
9844 return DAG.getMergeValues({Pred, Load.getValue(1)}, dl);
9845}
9846
9847void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
9848 SelectionDAG &DAG) const {
9849 LoadSDNode *LD = cast<LoadSDNode>(N);
9850 EVT MemVT = LD->getMemoryVT();
9851
9852 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9853 !Subtarget->isThumb1Only() && LD->isVolatile() &&
9854 LD->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
9855 assert(LD->isUnindexed() && "Loads should be unindexed at this point.");
9856 SDLoc dl(N);
9858 ARMISD::LDRD, dl, DAG.getVTList({MVT::i32, MVT::i32, MVT::Other}),
9859 {LD->getChain(), LD->getBasePtr()}, MemVT, LD->getMemOperand());
9860 SDValue Lo = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 0 : 1);
9861 SDValue Hi = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 1 : 0);
9862 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
9863 Results.append({Pair, Result.getValue(2)});
9864 } else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
9865 auto Pair = LowerAEABIUnalignedLoad(SDValue(N, 0), DAG);
9866 if (Pair.first) {
9867 Results.push_back(Pair.first);
9868 Results.push_back(Pair.second);
9869 }
9870 }
9871}
9872
9874 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
9875 EVT MemVT = ST->getMemoryVT();
9876 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9877 MemVT == MVT::v16i1) &&
9878 "Expected a predicate type!");
9879 assert(MemVT == ST->getValue().getValueType());
9880 assert(!ST->isTruncatingStore() && "Expected a non-extending store");
9881 assert(ST->isUnindexed() && "Expected a unindexed store");
9882
9883 // Only store the v2i1 or v4i1 or v8i1 worth of bits, via a buildvector with
9884 // top bits unset and a scalar store.
9885 SDLoc dl(Op);
9886 SDValue Build = ST->getValue();
9887 if (MemVT != MVT::v16i1) {
9889 for (unsigned I = 0; I < MemVT.getVectorNumElements(); I++) {
9890 unsigned Elt = DAG.getDataLayout().isBigEndian()
9891 ? MemVT.getVectorNumElements() - I - 1
9892 : I;
9893 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Build,
9894 DAG.getConstant(Elt, dl, MVT::i32)));
9895 }
9896 for (unsigned I = MemVT.getVectorNumElements(); I < 16; I++)
9897 Ops.push_back(DAG.getUNDEF(MVT::i32));
9898 Build = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i1, Ops);
9899 }
9900 SDValue GRP = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Build);
9901 if (MemVT == MVT::v16i1 && DAG.getDataLayout().isBigEndian())
9902 GRP = DAG.getNode(ISD::SRL, dl, MVT::i32,
9903 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, GRP),
9904 DAG.getConstant(16, dl, MVT::i32));
9905 return DAG.getTruncStore(
9906 ST->getChain(), dl, GRP, ST->getBasePtr(),
9908 ST->getMemOperand());
9909}
9910
9911SDValue ARMTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG,
9912 const ARMSubtarget *Subtarget) const {
9913 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
9914 EVT MemVT = ST->getMemoryVT();
9915
9916 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9917 !Subtarget->isThumb1Only() && ST->isVolatile() &&
9918 ST->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
9919 assert(ST->isUnindexed() && "Stores should be unindexed at this point.");
9920 SDNode *N = Op.getNode();
9921 SDLoc dl(N);
9922
9923 SDValue Lo = DAG.getNode(
9924 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(),
9925 DAG.getTargetConstant(DAG.getDataLayout().isLittleEndian() ? 0 : 1, dl,
9926 MVT::i32));
9927 SDValue Hi = DAG.getNode(
9928 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(),
9929 DAG.getTargetConstant(DAG.getDataLayout().isLittleEndian() ? 1 : 0, dl,
9930 MVT::i32));
9931
9932 return DAG.getMemIntrinsicNode(ARMISD::STRD, dl, DAG.getVTList(MVT::Other),
9933 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
9934 MemVT, ST->getMemOperand());
9935 } else if (Subtarget->hasMVEIntegerOps() &&
9936 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9937 MemVT == MVT::v16i1))) {
9938 return LowerPredicateStore(Op, DAG);
9939 } else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
9940 return LowerAEABIUnalignedStore(Op, DAG);
9941 }
9942 return SDValue();
9943}
9944
9945static bool isZeroVector(SDValue N) {
9946 return (ISD::isBuildVectorAllZeros(N) || (N->getOpcode() == ARMISD::VMOVIMM &&
9947 isNullConstant(N->getOperand(0))));
9948}
9949
9952 MVT VT = Op.getSimpleValueType();
9953 SDValue Mask = N->getMask();
9954 SDValue PassThru = N->getPassThru();
9955 SDLoc dl(Op);
9956
9957 if (isZeroVector(PassThru))
9958 return Op;
9959
9960 // MVE Masked loads use zero as the passthru value. Here we convert undef to
9961 // zero too, and other values are lowered to a select.
9962 SDValue ZeroVec = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
9963 DAG.getTargetConstant(0, dl, MVT::i32));
9964 SDValue NewLoad = DAG.getMaskedLoad(
9965 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, ZeroVec,
9966 N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
9967 N->getExtensionType(), N->isExpandingLoad());
9968 SDValue Combo = NewLoad;
9969 bool PassThruIsCastZero = (PassThru.getOpcode() == ISD::BITCAST ||
9970 PassThru.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
9971 isZeroVector(PassThru->getOperand(0));
9972 if (!PassThru.isUndef() && !PassThruIsCastZero)
9973 Combo = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
9974 return DAG.getMergeValues({Combo, NewLoad.getValue(1)}, dl);
9975}
9976
9978 const ARMSubtarget *ST) {
9979 if (!ST->hasMVEIntegerOps())
9980 return SDValue();
9981
9982 SDLoc dl(Op);
9983 unsigned BaseOpcode = 0;
9984 switch (Op->getOpcode()) {
9985 default: llvm_unreachable("Expected VECREDUCE opcode");
9986 case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break;
9987 case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break;
9988 case ISD::VECREDUCE_MUL: BaseOpcode = ISD::MUL; break;
9989 case ISD::VECREDUCE_AND: BaseOpcode = ISD::AND; break;
9990 case ISD::VECREDUCE_OR: BaseOpcode = ISD::OR; break;
9991 case ISD::VECREDUCE_XOR: BaseOpcode = ISD::XOR; break;
9992 case ISD::VECREDUCE_FMAX: BaseOpcode = ISD::FMAXNUM; break;
9993 case ISD::VECREDUCE_FMIN: BaseOpcode = ISD::FMINNUM; break;
9994 }
9995
9996 SDValue Op0 = Op->getOperand(0);
9997 EVT VT = Op0.getValueType();
9998 EVT EltVT = VT.getVectorElementType();
9999 unsigned NumElts = VT.getVectorNumElements();
10000 unsigned NumActiveLanes = NumElts;
10001
10002 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10003 NumActiveLanes == 2) &&
10004 "Only expected a power 2 vector size");
10005
10006 // Use Mul(X, Rev(X)) until 4 items remain. Going down to 4 vector elements
10007 // allows us to easily extract vector elements from the lanes.
10008 while (NumActiveLanes > 4) {
10009 unsigned RevOpcode = NumActiveLanes == 16 ? ARMISD::VREV16 : ARMISD::VREV32;
10010 SDValue Rev = DAG.getNode(RevOpcode, dl, VT, Op0);
10011 Op0 = DAG.getNode(BaseOpcode, dl, VT, Op0, Rev);
10012 NumActiveLanes /= 2;
10013 }
10014
10015 SDValue Res;
10016 if (NumActiveLanes == 4) {
10017 // The remaining 4 elements are summed sequentially
10018 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10019 DAG.getConstant(0 * NumElts / 4, dl, MVT::i32));
10020 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10021 DAG.getConstant(1 * NumElts / 4, dl, MVT::i32));
10022 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10023 DAG.getConstant(2 * NumElts / 4, dl, MVT::i32));
10024 SDValue Ext3 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10025 DAG.getConstant(3 * NumElts / 4, dl, MVT::i32));
10026 SDValue Res0 = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
10027 SDValue Res1 = DAG.getNode(BaseOpcode, dl, EltVT, Ext2, Ext3, Op->getFlags());
10028 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res0, Res1, Op->getFlags());
10029 } else {
10030 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10031 DAG.getConstant(0, dl, MVT::i32));
10032 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10033 DAG.getConstant(1, dl, MVT::i32));
10034 Res = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
10035 }
10036
10037 // Result type may be wider than element type.
10038 if (EltVT != Op->getValueType(0))
10039 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Op->getValueType(0), Res);
10040 return Res;
10041}
10042
10044 const ARMSubtarget *ST) {
10045 if (!ST->hasMVEFloatOps())
10046 return SDValue();
10047 return LowerVecReduce(Op, DAG, ST);
10048}
10049
10051 const ARMSubtarget *ST) {
10052 if (!ST->hasNEON())
10053 return SDValue();
10054
10055 SDLoc dl(Op);
10056 SDValue Op0 = Op->getOperand(0);
10057 EVT VT = Op0.getValueType();
10058 EVT EltVT = VT.getVectorElementType();
10059
10060 unsigned PairwiseIntrinsic = 0;
10061 switch (Op->getOpcode()) {
10062 default:
10063 llvm_unreachable("Expected VECREDUCE opcode");
10065 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10066 break;
10068 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10069 break;
10071 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10072 break;
10074 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10075 break;
10076 }
10077 SDValue PairwiseOp = DAG.getConstant(PairwiseIntrinsic, dl, MVT::i32);
10078
10079 unsigned NumElts = VT.getVectorNumElements();
10080 unsigned NumActiveLanes = NumElts;
10081
10082 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10083 NumActiveLanes == 2) &&
10084 "Only expected a power 2 vector size");
10085
10086 // Split 128-bit vectors, since vpmin/max takes 2 64-bit vectors.
10087 if (VT.is128BitVector()) {
10088 SDValue Lo, Hi;
10089 std::tie(Lo, Hi) = DAG.SplitVector(Op0, dl);
10090 VT = Lo.getValueType();
10091 Op0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, {PairwiseOp, Lo, Hi});
10092 NumActiveLanes /= 2;
10093 }
10094
10095 // Use pairwise reductions until one lane remains
10096 while (NumActiveLanes > 1) {
10097 Op0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, {PairwiseOp, Op0, Op0});
10098 NumActiveLanes /= 2;
10099 }
10100
10101 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10102 DAG.getConstant(0, dl, MVT::i32));
10103
10104 // Result type may be wider than element type.
10105 if (EltVT != Op.getValueType()) {
10106 unsigned Extend = 0;
10107 switch (Op->getOpcode()) {
10108 default:
10109 llvm_unreachable("Expected VECREDUCE opcode");
10112 Extend = ISD::ZERO_EXTEND;
10113 break;
10116 Extend = ISD::SIGN_EXTEND;
10117 break;
10118 }
10119 Res = DAG.getNode(Extend, dl, Op.getValueType(), Res);
10120 }
10121 return Res;
10122}
10123
10126 SelectionDAG &DAG,
10127 const ARMSubtarget *Subtarget) {
10128 SDLoc DL(N);
10129 // Under Power Management extensions, the cycle-count is:
10130 // mrc p15, #0, <Rt>, c9, c13, #0
10131 SDValue Ops[] = { N->getOperand(0), // Chain
10132 DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
10133 DAG.getTargetConstant(15, DL, MVT::i32),
10134 DAG.getTargetConstant(0, DL, MVT::i32),
10135 DAG.getTargetConstant(9, DL, MVT::i32),
10136 DAG.getTargetConstant(13, DL, MVT::i32),
10137 DAG.getTargetConstant(0, DL, MVT::i32)
10138 };
10139
10140 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
10141 DAG.getVTList(MVT::i32, MVT::Other), Ops);
10142 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Cycles32,
10143 DAG.getConstant(0, DL, MVT::i32)));
10144 Results.push_back(Cycles32.getValue(1));
10145}
10146
10148 SDValue V1) {
10149 SDLoc dl(V0.getNode());
10150 SDValue RegClass =
10151 DAG.getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
10152 SDValue SubReg0 = DAG.getTargetConstant(ARM::gsub_0, dl, MVT::i32);
10153 SDValue SubReg1 = DAG.getTargetConstant(ARM::gsub_1, dl, MVT::i32);
10154 const SDValue Ops[] = {RegClass, V0, SubReg0, V1, SubReg1};
10155 return SDValue(
10156 DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10157}
10158
10160 SDLoc dl(V.getNode());
10161 auto [VLo, VHi] = DAG.SplitScalar(V, dl, MVT::i32, MVT::i32);
10162 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10163 if (isBigEndian)
10164 std::swap(VLo, VHi);
10165 return createGPRPairNode2xi32(DAG, VLo, VHi);
10166}
10167
10170 SelectionDAG &DAG) {
10171 assert(N->getValueType(0) == MVT::i64 &&
10172 "AtomicCmpSwap on types less than 64 should be legal");
10173 SDValue Ops[] = {
10174 createGPRPairNode2xi32(DAG, N->getOperand(1),
10175 DAG.getUNDEF(MVT::i32)), // pointer, temp
10176 createGPRPairNodei64(DAG, N->getOperand(2)), // expected
10177 createGPRPairNodei64(DAG, N->getOperand(3)), // new
10178 N->getOperand(0), // chain in
10179 };
10180 SDNode *CmpSwap = DAG.getMachineNode(
10181 ARM::CMP_SWAP_64, SDLoc(N),
10182 DAG.getVTList(MVT::Untyped, MVT::Untyped, MVT::Other), Ops);
10183
10184 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
10185 DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
10186
10187 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10188
10189 SDValue Lo =
10190 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_1 : ARM::gsub_0,
10191 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0));
10192 SDValue Hi =
10193 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_0 : ARM::gsub_1,
10194 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0));
10195 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i64, Lo, Hi));
10196 Results.push_back(SDValue(CmpSwap, 2));
10197}
10198
10199SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
10200 SDLoc dl(Op);
10201 EVT VT = Op.getValueType();
10202 SDValue Chain = Op.getOperand(0);
10203 SDValue LHS = Op.getOperand(1);
10204 SDValue RHS = Op.getOperand(2);
10205 ISD::CondCode CC = Op.getOperandCondCode(3);
10206 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
10207
10208 // If we don't have instructions of this float type then soften to a libcall
10209 // and use SETCC instead.
10210 if (isUnsupportedFloatingType(LHS.getValueType())) {
10211 softenSetCCOperands(DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS,
10212 Chain, IsSignaling);
10213 if (!RHS.getNode()) {
10214 RHS = DAG.getConstant(0, dl, LHS.getValueType());
10215 CC = ISD::SETNE;
10216 }
10217 SDValue Result = DAG.getNode(ISD::SETCC, dl, VT, LHS, RHS,
10218 DAG.getCondCode(CC));
10219 return DAG.getMergeValues({Result, Chain}, dl);
10220 }
10221
10222 ARMCC::CondCodes CondCode, CondCode2;
10223 FPCCToARMCC(CC, CondCode, CondCode2);
10224
10225 SDValue True = DAG.getConstant(1, dl, VT);
10226 SDValue False = DAG.getConstant(0, dl, VT);
10227 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
10228 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10229 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, Cmp, DAG);
10230 if (CondCode2 != ARMCC::AL) {
10231 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
10232 Result = getCMOV(dl, VT, Result, True, ARMcc, Cmp, DAG);
10233 }
10234 return DAG.getMergeValues({Result, Chain}, dl);
10235}
10236
10237SDValue ARMTargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
10238 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
10239
10240 EVT VT = getPointerTy(DAG.getDataLayout());
10241 int FI = MFI.CreateFixedObject(4, 0, false);
10242 return DAG.getFrameIndex(FI, VT);
10243}
10244
10245SDValue ARMTargetLowering::LowerFP_TO_BF16(SDValue Op,
10246 SelectionDAG &DAG) const {
10247 SDLoc DL(Op);
10248 MakeLibCallOptions CallOptions;
10249 MVT SVT = Op.getOperand(0).getSimpleValueType();
10250 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, MVT::bf16);
10251 SDValue Res =
10252 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
10253 return DAG.getBitcast(MVT::i32, Res);
10254}
10255
10256SDValue ARMTargetLowering::LowerCMP(SDValue Op, SelectionDAG &DAG) const {
10257 SDLoc dl(Op);
10258 SDValue LHS = Op.getOperand(0);
10259 SDValue RHS = Op.getOperand(1);
10260
10261 // Determine if this is signed or unsigned comparison
10262 bool IsSigned = (Op.getOpcode() == ISD::SCMP);
10263
10264 // Special case for Thumb1 UCMP only
10265 if (!IsSigned && Subtarget->isThumb1Only()) {
10266 // For Thumb unsigned comparison, use this sequence:
10267 // subs r2, r0, r1 ; r2 = LHS - RHS, sets flags
10268 // sbc r2, r2 ; r2 = r2 - r2 - !carry
10269 // cmp r1, r0 ; compare RHS with LHS
10270 // sbc r1, r1 ; r1 = r1 - r1 - !carry
10271 // subs r0, r2, r1 ; r0 = r2 - r1 (final result)
10272
10273 // First subtraction: LHS - RHS
10274 SDValue Sub1WithFlags = DAG.getNode(
10275 ARMISD::SUBC, dl, DAG.getVTList(MVT::i32, FlagsVT), LHS, RHS);
10276 SDValue Sub1Result = Sub1WithFlags.getValue(0);
10277 SDValue Flags1 = Sub1WithFlags.getValue(1);
10278
10279 // SUBE: Sub1Result - Sub1Result - !carry
10280 // This gives 0 if LHS >= RHS (unsigned), -1 if LHS < RHS (unsigned)
10281 SDValue Sbc1 =
10282 DAG.getNode(ARMISD::SUBE, dl, DAG.getVTList(MVT::i32, FlagsVT),
10283 Sub1Result, Sub1Result, Flags1);
10284 SDValue Sbc1Result = Sbc1.getValue(0);
10285
10286 // Second comparison: RHS vs LHS (reverse comparison)
10287 SDValue CmpFlags = DAG.getNode(ARMISD::CMP, dl, FlagsVT, RHS, LHS);
10288
10289 // SUBE: RHS - RHS - !carry
10290 // This gives 0 if RHS <= LHS (unsigned), -1 if RHS > LHS (unsigned)
10291 SDValue Sbc2 = DAG.getNode(
10292 ARMISD::SUBE, dl, DAG.getVTList(MVT::i32, FlagsVT), RHS, RHS, CmpFlags);
10293 SDValue Sbc2Result = Sbc2.getValue(0);
10294
10295 // Final subtraction: Sbc1Result - Sbc2Result (no flags needed)
10296 SDValue Result =
10297 DAG.getNode(ISD::SUB, dl, MVT::i32, Sbc1Result, Sbc2Result);
10298 if (Op.getValueType() != MVT::i32)
10299 Result = DAG.getSExtOrTrunc(Result, dl, Op.getValueType());
10300
10301 return Result;
10302 }
10303
10304 // For the ARM assembly pattern:
10305 // subs r0, r0, r1 ; subtract RHS from LHS and set flags
10306 // movgt r0, #1 ; if LHS > RHS, set result to 1 (GT for signed, HI for
10307 // unsigned) mvnlt r0, #0 ; if LHS < RHS, set result to -1 (LT for
10308 // signed, LO for unsigned)
10309 // ; if LHS == RHS, result remains 0 from the subs
10310
10311 // Optimization: if RHS is a subtraction against 0, use ADDC instead of SUBC
10312 unsigned Opcode = ARMISD::SUBC;
10313
10314 // Check if RHS is a subtraction against 0: (0 - X)
10315 if (RHS.getOpcode() == ISD::SUB) {
10316 SDValue SubLHS = RHS.getOperand(0);
10317 SDValue SubRHS = RHS.getOperand(1);
10318
10319 // Check if it's 0 - X
10320 if (isNullConstant(SubLHS)) {
10321 bool CanUseAdd = false;
10322 if (IsSigned) {
10323 // For SCMP: only if X is known to never be INT_MIN (to avoid overflow)
10324 if (RHS->getFlags().hasNoSignedWrap() || !DAG.computeKnownBits(SubRHS)
10326 .isMinSignedValue()) {
10327 CanUseAdd = true;
10328 }
10329 } else {
10330 // For UCMP: only if X is known to never be zero
10331 if (DAG.isKnownNeverZero(SubRHS)) {
10332 CanUseAdd = true;
10333 }
10334 }
10335
10336 if (CanUseAdd) {
10337 Opcode = ARMISD::ADDC;
10338 RHS = SubRHS; // Replace RHS with X, so we do LHS + X instead of
10339 // LHS - (0 - X)
10340 }
10341 }
10342 }
10343
10344 // Generate the operation with flags
10345 SDValue OpWithFlags =
10346 DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, FlagsVT), LHS, RHS);
10347
10348 SDValue OpResult = OpWithFlags.getValue(0);
10349 SDValue Flags = OpWithFlags.getValue(1);
10350
10351 // Constants for conditional moves
10352 SDValue One = DAG.getConstant(1, dl, MVT::i32);
10353 SDValue MinusOne = DAG.getAllOnesConstant(dl, MVT::i32);
10354
10355 // Select condition codes based on signed vs unsigned
10356 ARMCC::CondCodes GTCond = IsSigned ? ARMCC::GT : ARMCC::HI;
10357 ARMCC::CondCodes LTCond = IsSigned ? ARMCC::LT : ARMCC::LO;
10358
10359 // First conditional move: if greater than, set to 1
10360 SDValue GTCondValue = DAG.getConstant(GTCond, dl, MVT::i32);
10361 SDValue Result1 = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, OpResult, One,
10362 GTCondValue, Flags);
10363
10364 // Second conditional move: if less than, set to -1
10365 SDValue LTCondValue = DAG.getConstant(LTCond, dl, MVT::i32);
10366 SDValue Result2 = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, Result1, MinusOne,
10367 LTCondValue, Flags);
10368
10369 if (Op.getValueType() != MVT::i32)
10370 Result2 = DAG.getSExtOrTrunc(Result2, dl, Op.getValueType());
10371
10372 return Result2;
10373}
10374
10376 LLVM_DEBUG(dbgs() << "Lowering node: "; Op.dump());
10377 switch (Op.getOpcode()) {
10378 default: llvm_unreachable("Don't know how to custom lower this!");
10379 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
10380 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
10381 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
10382 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
10383 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
10384 case ISD::SELECT: return LowerSELECT(Op, DAG);
10385 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
10386 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
10387 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
10388 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
10389 case ISD::VASTART: return LowerVASTART(Op, DAG);
10390 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
10391 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
10394 case ISD::SINT_TO_FP:
10395 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
10398 case ISD::FP_TO_SINT:
10399 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
10401 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG, Subtarget);
10402 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
10403 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
10404 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
10405 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
10406 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
10407 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
10408 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG, Subtarget);
10409 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
10410 Subtarget);
10411 case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG, Subtarget);
10412 case ISD::SHL:
10413 case ISD::SRL:
10414 case ISD::SRA: return LowerShift(Op.getNode(), DAG, Subtarget);
10415 case ISD::SREM: return LowerREM(Op.getNode(), DAG);
10416 case ISD::UREM: return LowerREM(Op.getNode(), DAG);
10417 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG);
10418 case ISD::SRL_PARTS:
10419 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
10420 case ISD::CTTZ:
10421 case ISD::CTTZ_ZERO_POISON: return LowerCTTZ(Op.getNode(), DAG, Subtarget);
10422 case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget);
10423 case ISD::SETCC: return LowerVSETCC(Op, DAG, Subtarget);
10424 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
10425 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget);
10426 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget);
10427 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
10428 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG, Subtarget);
10429 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
10430 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG, Subtarget);
10431 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG, Subtarget);
10432 case ISD::TRUNCATE: return LowerTruncate(Op.getNode(), DAG, Subtarget);
10433 case ISD::SIGN_EXTEND:
10434 case ISD::ZERO_EXTEND: return LowerVectorExtend(Op.getNode(), DAG, Subtarget);
10435 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
10436 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG);
10437 case ISD::SET_FPMODE:
10438 return LowerSET_FPMODE(Op, DAG);
10439 case ISD::RESET_FPMODE:
10440 return LowerRESET_FPMODE(Op, DAG);
10441 case ISD::MUL: return LowerMUL(Op, DAG);
10442 case ISD::SDIV:
10443 if (getTargetMachine().getTargetTriple().isOSWindows() &&
10444 !Op.getValueType().isVector())
10445 return LowerDIV_Windows(Op, DAG, /* Signed */ true);
10446 return LowerSDIV(Op, DAG, Subtarget);
10447 case ISD::UDIV:
10448 if (getTargetMachine().getTargetTriple().isOSWindows() &&
10449 !Op.getValueType().isVector())
10450 return LowerDIV_Windows(Op, DAG, /* Signed */ false);
10451 return LowerUDIV(Op, DAG, Subtarget);
10452 case ISD::UADDO_CARRY:
10453 return LowerADDSUBO_CARRY(Op, DAG, ARMISD::ADDE, false /*unsigned*/);
10454 case ISD::USUBO_CARRY:
10455 return LowerADDSUBO_CARRY(Op, DAG, ARMISD::SUBE, false /*unsigned*/);
10456 case ISD::SADDO_CARRY:
10457 return LowerADDSUBO_CARRY(Op, DAG, ARMISD::ADDE, true /*signed*/);
10458 case ISD::SSUBO_CARRY:
10459 return LowerADDSUBO_CARRY(Op, DAG, ARMISD::SUBE, true /*signed*/);
10460 case ISD::UADDO:
10461 case ISD::USUBO:
10462 case ISD::UMULO:
10463 case ISD::SADDO:
10464 case ISD::SSUBO:
10465 case ISD::SMULO:
10466 return LowerALUO(Op, DAG);
10467 case ISD::SADDSAT:
10468 case ISD::SSUBSAT:
10469 case ISD::UADDSAT:
10470 case ISD::USUBSAT:
10471 return LowerADDSUBSAT(Op, DAG, Subtarget);
10472 case ISD::LOAD: {
10473 auto *LD = cast<LoadSDNode>(Op);
10474 EVT MemVT = LD->getMemoryVT();
10475 if (Subtarget->hasMVEIntegerOps() &&
10476 (MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10477 MemVT == MVT::v16i1))
10478 return LowerPredicateLoad(Op, DAG);
10479
10480 auto Pair = LowerAEABIUnalignedLoad(Op, DAG);
10481 if (Pair.first)
10482 return DAG.getMergeValues({Pair.first, Pair.second}, SDLoc(Pair.first));
10483 return SDValue();
10484 }
10485 case ISD::STORE:
10486 return LowerSTORE(Op, DAG, Subtarget);
10487 case ISD::MLOAD:
10488 return LowerMLOAD(Op, DAG);
10489 case ISD::VECREDUCE_MUL:
10490 case ISD::VECREDUCE_AND:
10491 case ISD::VECREDUCE_OR:
10492 case ISD::VECREDUCE_XOR:
10493 return LowerVecReduce(Op, DAG, Subtarget);
10498 return LowerVecReduceF(Op, DAG, Subtarget);
10503 return LowerVecReduceMinMax(Op, DAG, Subtarget);
10504 case ISD::SDIVREM:
10505 case ISD::UDIVREM: return LowerDivRem(Op, DAG);
10507 if (getTargetMachine().getTargetTriple().isOSWindows())
10508 return LowerDYNAMIC_STACKALLOC(Op, DAG);
10509 llvm_unreachable("Don't know how to custom lower this!");
10511 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG);
10513 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
10514 case ISD::STRICT_FSETCC:
10515 case ISD::STRICT_FSETCCS: return LowerFSETCC(Op, DAG);
10516 case ISD::SPONENTRY:
10517 return LowerSPONENTRY(Op, DAG);
10518 case ISD::FP_TO_BF16:
10519 return LowerFP_TO_BF16(Op, DAG);
10520 case ARMISD::WIN__DBZCHK: return SDValue();
10521 case ISD::UCMP:
10522 case ISD::SCMP:
10523 return LowerCMP(Op, DAG);
10524 case ISD::ABS:
10525 return LowerABS(Op, DAG);
10526 case ISD::STRICT_LROUND:
10528 case ISD::STRICT_LRINT:
10529 case ISD::STRICT_LLRINT: {
10530 assert((Op.getOperand(1).getValueType() == MVT::f16 ||
10531 Op.getOperand(1).getValueType() == MVT::bf16) &&
10532 "Expected custom lowering of rounding operations only for f16");
10533 SDLoc DL(Op);
10534 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
10535 {Op.getOperand(0), Op.getOperand(1)});
10536 return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
10537 {Ext.getValue(1), Ext.getValue(0)});
10538 }
10539 }
10540}
10541
10543 SelectionDAG &DAG) {
10544 unsigned IntNo = N->getConstantOperandVal(0);
10545 unsigned Opc = 0;
10546 if (IntNo == Intrinsic::arm_smlald)
10547 Opc = ARMISD::SMLALD;
10548 else if (IntNo == Intrinsic::arm_smlaldx)
10549 Opc = ARMISD::SMLALDX;
10550 else if (IntNo == Intrinsic::arm_smlsld)
10551 Opc = ARMISD::SMLSLD;
10552 else if (IntNo == Intrinsic::arm_smlsldx)
10553 Opc = ARMISD::SMLSLDX;
10554 else
10555 return;
10556
10557 SDLoc dl(N);
10558 SDValue Lo, Hi;
10559 std::tie(Lo, Hi) = DAG.SplitScalar(N->getOperand(3), dl, MVT::i32, MVT::i32);
10560
10561 SDValue LongMul = DAG.getNode(Opc, dl,
10562 DAG.getVTList(MVT::i32, MVT::i32),
10563 N->getOperand(1), N->getOperand(2),
10564 Lo, Hi);
10565 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64,
10566 LongMul.getValue(0), LongMul.getValue(1)));
10567}
10568
10569/// ReplaceNodeResults - Replace the results of node with an illegal result
10570/// type with new values built out of custom code.
10573 SelectionDAG &DAG) const {
10574 SDValue Res;
10575 switch (N->getOpcode()) {
10576 default:
10577 llvm_unreachable("Don't know how to custom expand this!");
10578 case ISD::READ_REGISTER:
10580 break;
10581 case ISD::BITCAST:
10582 Res = ExpandBITCAST(N, DAG, Subtarget);
10583 break;
10584 case ISD::SRL:
10585 case ISD::SRA:
10586 case ISD::SHL:
10587 Res = Expand64BitShift(N, DAG, Subtarget);
10588 break;
10589 case ISD::SREM:
10590 case ISD::UREM:
10591 Res = LowerREM(N, DAG);
10592 break;
10593 case ISD::SDIVREM:
10594 case ISD::UDIVREM:
10595 Res = LowerDivRem(SDValue(N, 0), DAG);
10596 assert(Res.getNumOperands() == 2 && "DivRem needs two values");
10597 Results.push_back(Res.getValue(0));
10598 Results.push_back(Res.getValue(1));
10599 return;
10600 case ISD::SADDSAT:
10601 case ISD::SSUBSAT:
10602 case ISD::UADDSAT:
10603 case ISD::USUBSAT:
10604 Res = LowerADDSUBSAT(SDValue(N, 0), DAG, Subtarget);
10605 break;
10607 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget);
10608 return;
10609 case ISD::UDIV:
10610 case ISD::SDIV:
10611 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
10612 "can only expand DIV on Windows");
10613 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV,
10614 Results);
10617 return;
10619 return ReplaceLongIntrinsic(N, Results, DAG);
10620 case ISD::LOAD:
10621 LowerLOAD(N, Results, DAG);
10622 break;
10623 case ISD::STORE:
10624 Res = LowerAEABIUnalignedStore(SDValue(N, 0), DAG);
10625 break;
10626 case ISD::TRUNCATE:
10627 Res = LowerTruncate(N, DAG, Subtarget);
10628 break;
10629 case ISD::SIGN_EXTEND:
10630 case ISD::ZERO_EXTEND:
10631 Res = LowerVectorExtend(N, DAG, Subtarget);
10632 break;
10635 Res = LowerFP_TO_INT_SAT(SDValue(N, 0), DAG, Subtarget);
10636 break;
10637 }
10638 if (Res.getNode())
10639 Results.push_back(Res);
10640}
10641
10642//===----------------------------------------------------------------------===//
10643// ARM Scheduler Hooks
10644//===----------------------------------------------------------------------===//
10645
10646/// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
10647/// registers the function context.
10648void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
10650 MachineBasicBlock *DispatchBB,
10651 int FI) const {
10652 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
10653 "ROPI/RWPI not currently supported with SjLj");
10654 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10655 DebugLoc dl = MI.getDebugLoc();
10656 MachineFunction *MF = MBB->getParent();
10660 const Function &F = MF->getFunction();
10661
10662 bool isThumb = Subtarget->isThumb();
10663 bool isThumb2 = Subtarget->isThumb2();
10664
10665 unsigned PCLabelId = AFI->createPICLabelUId();
10666 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8;
10668 ARMConstantPoolMBB::Create(F.getContext(), DispatchBB, PCLabelId, PCAdj);
10669 unsigned CPI = MCP->getConstantPoolIndex(CPV, Align(4));
10670
10671 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass
10672 : &ARM::GPRRegClass;
10673
10674 // Grab constant pool and fixed stack memory operands.
10675 MachineMemOperand *CPMMO =
10678
10679 MachineMemOperand *FIMMOSt =
10682
10683 // Load the address of the dispatch MBB into the jump buffer.
10684 if (isThumb2) {
10685 // Incoming value: jbuf
10686 // ldr.n r5, LCPI1_1
10687 // orr r5, r5, #1
10688 // add r5, pc
10689 // str r5, [$jbuf, #+4] ; &jbuf[1]
10690 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10691 BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1)
10693 .addMemOperand(CPMMO)
10695 // Set the low bit because of thumb mode.
10696 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10697 BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2)
10698 .addReg(NewVReg1)
10699 .addImm(0x01)
10701 .add(condCodeOp());
10702 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10703 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3)
10704 .addReg(NewVReg2)
10705 .addImm(PCLabelId);
10706 BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12))
10707 .addReg(NewVReg3)
10708 .addFrameIndex(FI)
10709 .addImm(36) // &jbuf[1] :: pc
10710 .addMemOperand(FIMMOSt)
10712 } else if (isThumb) {
10713 // Incoming value: jbuf
10714 // ldr.n r1, LCPI1_4
10715 // add r1, pc
10716 // mov r2, #1
10717 // orrs r1, r2
10718 // add r2, $jbuf, #+4 ; &jbuf[1]
10719 // str r1, [r2]
10720 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10721 BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1)
10723 .addMemOperand(CPMMO)
10725 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10726 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2)
10727 .addReg(NewVReg1)
10728 .addImm(PCLabelId);
10729 // Set the low bit because of thumb mode.
10730 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10731 BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3)
10732 .addReg(ARM::CPSR, RegState::Define)
10733 .addImm(1)
10735 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10736 BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4)
10737 .addReg(ARM::CPSR, RegState::Define)
10738 .addReg(NewVReg2)
10739 .addReg(NewVReg3)
10741 Register NewVReg5 = MRI->createVirtualRegister(TRC);
10742 BuildMI(*MBB, MI, dl, TII->get(ARM::tADDframe), NewVReg5)
10743 .addFrameIndex(FI)
10744 .addImm(36); // &jbuf[1] :: pc
10745 BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi))
10746 .addReg(NewVReg4)
10747 .addReg(NewVReg5)
10748 .addImm(0)
10749 .addMemOperand(FIMMOSt)
10751 } else {
10752 // Incoming value: jbuf
10753 // ldr r1, LCPI1_1
10754 // add r1, pc, r1
10755 // str r1, [$jbuf, #+4] ; &jbuf[1]
10756 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10757 BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12), NewVReg1)
10759 .addImm(0)
10760 .addMemOperand(CPMMO)
10762 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10763 BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2)
10764 .addReg(NewVReg1)
10765 .addImm(PCLabelId)
10767 BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12))
10768 .addReg(NewVReg2)
10769 .addFrameIndex(FI)
10770 .addImm(36) // &jbuf[1] :: pc
10771 .addMemOperand(FIMMOSt)
10773 }
10774}
10775
10776void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
10777 MachineBasicBlock *MBB) const {
10778 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10779 DebugLoc dl = MI.getDebugLoc();
10780 MachineFunction *MF = MBB->getParent();
10781 MachineRegisterInfo *MRI = &MF->getRegInfo();
10782 MachineFrameInfo &MFI = MF->getFrameInfo();
10783 int FI = MFI.getFunctionContextIndex();
10784
10785 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass
10786 : &ARM::GPRnopcRegClass;
10787
10788 // Get a mapping of the call site numbers to all of the landing pads they're
10789 // associated with.
10790 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad;
10791 unsigned MaxCSNum = 0;
10792 for (MachineBasicBlock &BB : *MF) {
10793 if (!BB.isEHPad())
10794 continue;
10795
10796 // FIXME: We should assert that the EH_LABEL is the first MI in the landing
10797 // pad.
10798 for (MachineInstr &II : BB) {
10799 if (!II.isEHLabel())
10800 continue;
10801
10802 MCSymbol *Sym = II.getOperand(0).getMCSymbol();
10803 if (!MF->hasCallSiteLandingPad(Sym)) continue;
10804
10805 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym);
10806 for (unsigned Idx : CallSiteIdxs) {
10807 CallSiteNumToLPad[Idx].push_back(&BB);
10808 MaxCSNum = std::max(MaxCSNum, Idx);
10809 }
10810 break;
10811 }
10812 }
10813
10814 // Get an ordered list of the machine basic blocks for the jump table.
10815 std::vector<MachineBasicBlock*> LPadList;
10816 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs;
10817 LPadList.reserve(CallSiteNumToLPad.size());
10818 for (unsigned I = 1; I <= MaxCSNum; ++I) {
10819 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I];
10820 for (MachineBasicBlock *MBB : MBBList) {
10821 LPadList.push_back(MBB);
10822 InvokeBBs.insert_range(MBB->predecessors());
10823 }
10824 }
10825
10826 assert(!LPadList.empty() &&
10827 "No landing pad destinations for the dispatch jump table!");
10828
10829 // Create the jump table and associated information.
10830 MachineJumpTableInfo *JTI =
10831 MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline);
10832 unsigned MJTI = JTI->createJumpTableIndex(LPadList);
10833
10834 // Create the MBBs for the dispatch code.
10835
10836 // Shove the dispatch's address into the return slot in the function context.
10837 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
10838 DispatchBB->setIsEHPad();
10839
10840 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
10841
10842 BuildMI(TrapBB, dl, TII->get(Subtarget->isThumb() ? ARM::tTRAP : ARM::TRAP));
10843 DispatchBB->addSuccessor(TrapBB);
10844
10845 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
10846 DispatchBB->addSuccessor(DispContBB);
10847
10848 // Insert and MBBs.
10849 MF->insert(MF->end(), DispatchBB);
10850 MF->insert(MF->end(), DispContBB);
10851 MF->insert(MF->end(), TrapBB);
10852
10853 // Insert code into the entry block that creates and registers the function
10854 // context.
10855 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI);
10856
10857 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
10860
10861 MachineInstrBuilder MIB;
10862 MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup));
10863
10864 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
10865 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
10866
10867 // Add a register mask with no preserved registers. This results in all
10868 // registers being marked as clobbered. This can't work if the dispatch block
10869 // is in a Thumb1 function and is linked with ARM code which uses the FP
10870 // registers, as there is no way to preserve the FP registers in Thumb1 mode.
10872
10873 bool IsPositionIndependent = isPositionIndependent();
10874 unsigned NumLPads = LPadList.size();
10875 if (Subtarget->isThumb2()) {
10876 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10877 BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1)
10878 .addFrameIndex(FI)
10879 .addImm(4)
10880 .addMemOperand(FIMMOLd)
10882
10883 if (NumLPads < 256) {
10884 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri))
10885 .addReg(NewVReg1)
10886 .addImm(LPadList.size())
10888 } else {
10889 Register VReg1 = MRI->createVirtualRegister(TRC);
10890 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVi16), VReg1)
10891 .addImm(NumLPads & 0xFFFF)
10893
10894 unsigned VReg2 = VReg1;
10895 if ((NumLPads & 0xFFFF0000) != 0) {
10896 VReg2 = MRI->createVirtualRegister(TRC);
10897 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVTi16), VReg2)
10898 .addReg(VReg1)
10899 .addImm(NumLPads >> 16)
10901 }
10902
10903 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPrr))
10904 .addReg(NewVReg1)
10905 .addReg(VReg2)
10907 }
10908
10909 BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc))
10910 .addMBB(TrapBB)
10912 .addReg(ARM::CPSR);
10913
10914 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10915 BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT), NewVReg3)
10916 .addJumpTableIndex(MJTI)
10918
10919 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10920 BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg4)
10921 .addReg(NewVReg3)
10922 .addReg(NewVReg1)
10925 .add(condCodeOp());
10926
10927 BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT))
10928 .addReg(NewVReg4)
10929 .addReg(NewVReg1)
10930 .addJumpTableIndex(MJTI);
10931 } else if (Subtarget->isThumb()) {
10932 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10933 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1)
10934 .addFrameIndex(FI)
10935 .addImm(1)
10936 .addMemOperand(FIMMOLd)
10938
10939 if (NumLPads < 256) {
10940 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8))
10941 .addReg(NewVReg1)
10942 .addImm(NumLPads)
10944 } else {
10945 MachineConstantPool *ConstantPool = MF->getConstantPool();
10946 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext());
10947 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
10948
10949 // MachineConstantPool wants an explicit alignment.
10950 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
10951 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
10952
10953 Register VReg1 = MRI->createVirtualRegister(TRC);
10954 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRpci))
10955 .addReg(VReg1, RegState::Define)
10958 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPr))
10959 .addReg(NewVReg1)
10960 .addReg(VReg1)
10962 }
10963
10964 BuildMI(DispatchBB, dl, TII->get(ARM::tBcc))
10965 .addMBB(TrapBB)
10967 .addReg(ARM::CPSR);
10968
10969 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10970 BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2)
10971 .addReg(ARM::CPSR, RegState::Define)
10972 .addReg(NewVReg1)
10973 .addImm(2)
10975
10976 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10977 BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3)
10978 .addJumpTableIndex(MJTI)
10980
10981 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10982 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4)
10983 .addReg(ARM::CPSR, RegState::Define)
10984 .addReg(NewVReg2)
10985 .addReg(NewVReg3)
10987
10988 MachineMemOperand *JTMMOLd =
10989 MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF),
10991
10992 Register NewVReg5 = MRI->createVirtualRegister(TRC);
10993 BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5)
10994 .addReg(NewVReg4)
10995 .addImm(0)
10996 .addMemOperand(JTMMOLd)
10998
10999 unsigned NewVReg6 = NewVReg5;
11000 if (IsPositionIndependent) {
11001 NewVReg6 = MRI->createVirtualRegister(TRC);
11002 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6)
11003 .addReg(ARM::CPSR, RegState::Define)
11004 .addReg(NewVReg5)
11005 .addReg(NewVReg3)
11007 }
11008
11009 BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr))
11010 .addReg(NewVReg6)
11011 .addJumpTableIndex(MJTI);
11012 } else {
11013 Register NewVReg1 = MRI->createVirtualRegister(TRC);
11014 BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1)
11015 .addFrameIndex(FI)
11016 .addImm(4)
11017 .addMemOperand(FIMMOLd)
11019
11020 if (NumLPads < 256) {
11021 BuildMI(DispatchBB, dl, TII->get(ARM::CMPri))
11022 .addReg(NewVReg1)
11023 .addImm(NumLPads)
11025 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
11026 Register VReg1 = MRI->createVirtualRegister(TRC);
11027 BuildMI(DispatchBB, dl, TII->get(ARM::MOVi16), VReg1)
11028 .addImm(NumLPads & 0xFFFF)
11030
11031 unsigned VReg2 = VReg1;
11032 if ((NumLPads & 0xFFFF0000) != 0) {
11033 VReg2 = MRI->createVirtualRegister(TRC);
11034 BuildMI(DispatchBB, dl, TII->get(ARM::MOVTi16), VReg2)
11035 .addReg(VReg1)
11036 .addImm(NumLPads >> 16)
11038 }
11039
11040 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
11041 .addReg(NewVReg1)
11042 .addReg(VReg2)
11044 } else {
11045 MachineConstantPool *ConstantPool = MF->getConstantPool();
11046 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext());
11047 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
11048
11049 // MachineConstantPool wants an explicit alignment.
11050 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11051 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11052
11053 Register VReg1 = MRI->createVirtualRegister(TRC);
11054 BuildMI(DispatchBB, dl, TII->get(ARM::LDRcp))
11055 .addReg(VReg1, RegState::Define)
11057 .addImm(0)
11059 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
11060 .addReg(NewVReg1)
11061 .addReg(VReg1)
11063 }
11064
11065 BuildMI(DispatchBB, dl, TII->get(ARM::Bcc))
11066 .addMBB(TrapBB)
11068 .addReg(ARM::CPSR);
11069
11070 Register NewVReg3 = MRI->createVirtualRegister(TRC);
11071 BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg3)
11072 .addReg(NewVReg1)
11075 .add(condCodeOp());
11076 Register NewVReg4 = MRI->createVirtualRegister(TRC);
11077 BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4)
11078 .addJumpTableIndex(MJTI)
11080
11081 MachineMemOperand *JTMMOLd =
11082 MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF),
11084 Register NewVReg5 = MRI->createVirtualRegister(TRC);
11085 BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5)
11086 .addReg(NewVReg3)
11087 .addReg(NewVReg4)
11088 .addImm(0)
11089 .addMemOperand(JTMMOLd)
11091
11092 if (IsPositionIndependent) {
11093 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd))
11094 .addReg(NewVReg5)
11095 .addReg(NewVReg4)
11096 .addJumpTableIndex(MJTI);
11097 } else {
11098 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr))
11099 .addReg(NewVReg5)
11100 .addJumpTableIndex(MJTI);
11101 }
11102 }
11103
11104 // Add the jump table entries as successors to the MBB.
11105 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
11106 for (MachineBasicBlock *CurMBB : LPadList) {
11107 if (SeenMBBs.insert(CurMBB).second)
11108 DispContBB->addSuccessor(CurMBB);
11109 }
11110
11111 // N.B. the order the invoke BBs are processed in doesn't matter here.
11112 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF);
11114 for (MachineBasicBlock *BB : InvokeBBs) {
11115
11116 // Remove the landing pad successor from the invoke block and replace it
11117 // with the new dispatch block.
11118 SmallVector<MachineBasicBlock*, 4> Successors(BB->successors());
11119 while (!Successors.empty()) {
11120 MachineBasicBlock *SMBB = Successors.pop_back_val();
11121 if (SMBB->isEHPad()) {
11122 BB->removeSuccessor(SMBB);
11123 MBBLPads.push_back(SMBB);
11124 }
11125 }
11126
11127 BB->addSuccessor(DispatchBB, BranchProbability::getZero());
11128 BB->normalizeSuccProbs();
11129
11130 // Find the invoke call and mark all of the callee-saved registers as
11131 // 'implicit defined' so that they're spilled. This prevents code from
11132 // moving instructions to before the EH block, where they will never be
11133 // executed.
11135 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
11136 if (!II->isCall()) continue;
11137
11138 DenseSet<unsigned> DefRegs;
11140 OI = II->operands_begin(), OE = II->operands_end();
11141 OI != OE; ++OI) {
11142 if (!OI->isReg()) continue;
11143 DefRegs.insert(OI->getReg());
11144 }
11145
11146 MachineInstrBuilder MIB(*MF, &*II);
11147
11148 for (unsigned i = 0; SavedRegs[i] != 0; ++i) {
11149 unsigned Reg = SavedRegs[i];
11150 if (Subtarget->isThumb2() &&
11151 !ARM::tGPRRegClass.contains(Reg) &&
11152 !ARM::hGPRRegClass.contains(Reg))
11153 continue;
11154 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11155 continue;
11156 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11157 continue;
11158 if (!DefRegs.contains(Reg))
11160 }
11161
11162 break;
11163 }
11164 }
11165
11166 // Mark all former landing pads as non-landing pads. The dispatch is the only
11167 // landing pad now.
11168 for (MachineBasicBlock *MBBLPad : MBBLPads)
11169 MBBLPad->setIsEHPad(false);
11170
11171 // The instruction is gone now.
11172 MI.eraseFromParent();
11173}
11174
11175static
11177 for (MachineBasicBlock *S : MBB->successors())
11178 if (S != Succ)
11179 return S;
11180 llvm_unreachable("Expecting a BB with two successors!");
11181}
11182
11183/// Return the load opcode for a given load size. If load size >= 8,
11184/// neon opcode will be returned.
11185static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) {
11186 if (LdSize >= 8)
11187 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11188 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11189 if (IsThumb1)
11190 return LdSize == 4 ? ARM::tLDRi
11191 : LdSize == 2 ? ARM::tLDRHi
11192 : LdSize == 1 ? ARM::tLDRBi : 0;
11193 if (IsThumb2)
11194 return LdSize == 4 ? ARM::t2LDR_POST
11195 : LdSize == 2 ? ARM::t2LDRH_POST
11196 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11197 return LdSize == 4 ? ARM::LDR_POST_IMM
11198 : LdSize == 2 ? ARM::LDRH_POST
11199 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11200}
11201
11202/// Return the store opcode for a given store size. If store size >= 8,
11203/// neon opcode will be returned.
11204static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) {
11205 if (StSize >= 8)
11206 return StSize == 16 ? ARM::VST1q32wb_fixed
11207 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11208 if (IsThumb1)
11209 return StSize == 4 ? ARM::tSTRi
11210 : StSize == 2 ? ARM::tSTRHi
11211 : StSize == 1 ? ARM::tSTRBi : 0;
11212 if (IsThumb2)
11213 return StSize == 4 ? ARM::t2STR_POST
11214 : StSize == 2 ? ARM::t2STRH_POST
11215 : StSize == 1 ? ARM::t2STRB_POST : 0;
11216 return StSize == 4 ? ARM::STR_POST_IMM
11217 : StSize == 2 ? ARM::STRH_POST
11218 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11219}
11220
11221/// Emit a post-increment load operation with given size. The instructions
11222/// will be added to BB at Pos.
11224 const TargetInstrInfo *TII, const DebugLoc &dl,
11225 unsigned LdSize, unsigned Data, unsigned AddrIn,
11226 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11227 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2);
11228 assert(LdOpc != 0 && "Should have a load opcode");
11229 if (LdSize >= 8) {
11230 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
11231 .addReg(AddrOut, RegState::Define)
11232 .addReg(AddrIn)
11233 .addImm(0)
11235 } else if (IsThumb1) {
11236 // load + update AddrIn
11237 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
11238 .addReg(AddrIn)
11239 .addImm(0)
11241 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut)
11242 .add(t1CondCodeOp())
11243 .addReg(AddrIn)
11244 .addImm(LdSize)
11246 } else if (IsThumb2) {
11247 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
11248 .addReg(AddrOut, RegState::Define)
11249 .addReg(AddrIn)
11250 .addImm(LdSize)
11252 } else { // arm
11253 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
11254 .addReg(AddrOut, RegState::Define)
11255 .addReg(AddrIn)
11256 .addReg(0)
11257 .addImm(LdSize)
11259 }
11260}
11261
11262/// Emit a post-increment store operation with given size. The instructions
11263/// will be added to BB at Pos.
11265 const TargetInstrInfo *TII, const DebugLoc &dl,
11266 unsigned StSize, unsigned Data, unsigned AddrIn,
11267 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11268 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2);
11269 assert(StOpc != 0 && "Should have a store opcode");
11270 if (StSize >= 8) {
11271 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
11272 .addReg(AddrIn)
11273 .addImm(0)
11274 .addReg(Data)
11276 } else if (IsThumb1) {
11277 // store + update AddrIn
11278 BuildMI(*BB, Pos, dl, TII->get(StOpc))
11279 .addReg(Data)
11280 .addReg(AddrIn)
11281 .addImm(0)
11283 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut)
11284 .add(t1CondCodeOp())
11285 .addReg(AddrIn)
11286 .addImm(StSize)
11288 } else if (IsThumb2) {
11289 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
11290 .addReg(Data)
11291 .addReg(AddrIn)
11292 .addImm(StSize)
11294 } else { // arm
11295 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
11296 .addReg(Data)
11297 .addReg(AddrIn)
11298 .addReg(0)
11299 .addImm(StSize)
11301 }
11302}
11303
11305ARMTargetLowering::EmitStructByval(MachineInstr &MI,
11306 MachineBasicBlock *BB) const {
11307 // This pseudo instruction has 3 operands: dst, src, size
11308 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold().
11309 // Otherwise, we will generate unrolled scalar copies.
11310 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11311 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11313
11314 Register dest = MI.getOperand(0).getReg();
11315 Register src = MI.getOperand(1).getReg();
11316 unsigned SizeVal = MI.getOperand(2).getImm();
11317 unsigned Alignment = MI.getOperand(3).getImm();
11318 DebugLoc dl = MI.getDebugLoc();
11319
11320 MachineFunction *MF = BB->getParent();
11321 MachineRegisterInfo &MRI = MF->getRegInfo();
11322 unsigned UnitSize = 0;
11323 const TargetRegisterClass *TRC = nullptr;
11324 const TargetRegisterClass *VecTRC = nullptr;
11325
11326 bool IsThumb1 = Subtarget->isThumb1Only();
11327 bool IsThumb2 = Subtarget->isThumb2();
11328 bool IsThumb = Subtarget->isThumb();
11329
11330 if (Alignment & 1) {
11331 UnitSize = 1;
11332 } else if (Alignment & 2) {
11333 UnitSize = 2;
11334 } else {
11335 // Check whether we can use NEON instructions.
11336 if (!MF->getFunction().hasFnAttribute(Attribute::NoImplicitFloat) &&
11337 Subtarget->hasNEON()) {
11338 if ((Alignment % 16 == 0) && SizeVal >= 16)
11339 UnitSize = 16;
11340 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11341 UnitSize = 8;
11342 }
11343 // Can't use NEON instructions.
11344 if (UnitSize == 0)
11345 UnitSize = 4;
11346 }
11347
11348 // Select the correct opcode and register class for unit size load/store
11349 bool IsNeon = UnitSize >= 8;
11350 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11351 if (IsNeon)
11352 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11353 : UnitSize == 8 ? &ARM::DPRRegClass
11354 : nullptr;
11355
11356 unsigned BytesLeft = SizeVal % UnitSize;
11357 unsigned LoopSize = SizeVal - BytesLeft;
11358
11359 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11360 // Use LDR and STR to copy.
11361 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize)
11362 // [destOut] = STR_POST(scratch, destIn, UnitSize)
11363 unsigned srcIn = src;
11364 unsigned destIn = dest;
11365 for (unsigned i = 0; i < LoopSize; i+=UnitSize) {
11366 Register srcOut = MRI.createVirtualRegister(TRC);
11367 Register destOut = MRI.createVirtualRegister(TRC);
11368 Register scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11369 emitPostLd(BB, MI, TII, dl, UnitSize, scratch, srcIn, srcOut,
11370 IsThumb1, IsThumb2);
11371 emitPostSt(BB, MI, TII, dl, UnitSize, scratch, destIn, destOut,
11372 IsThumb1, IsThumb2);
11373 srcIn = srcOut;
11374 destIn = destOut;
11375 }
11376
11377 // Handle the leftover bytes with LDRB and STRB.
11378 // [scratch, srcOut] = LDRB_POST(srcIn, 1)
11379 // [destOut] = STRB_POST(scratch, destIn, 1)
11380 for (unsigned i = 0; i < BytesLeft; i++) {
11381 Register srcOut = MRI.createVirtualRegister(TRC);
11382 Register destOut = MRI.createVirtualRegister(TRC);
11383 Register scratch = MRI.createVirtualRegister(TRC);
11384 emitPostLd(BB, MI, TII, dl, 1, scratch, srcIn, srcOut,
11385 IsThumb1, IsThumb2);
11386 emitPostSt(BB, MI, TII, dl, 1, scratch, destIn, destOut,
11387 IsThumb1, IsThumb2);
11388 srcIn = srcOut;
11389 destIn = destOut;
11390 }
11391 MI.eraseFromParent(); // The instruction is gone now.
11392 return BB;
11393 }
11394
11395 // Expand the pseudo op to a loop.
11396 // thisMBB:
11397 // ...
11398 // movw varEnd, # --> with thumb2
11399 // movt varEnd, #
11400 // ldrcp varEnd, idx --> without thumb2
11401 // fallthrough --> loopMBB
11402 // loopMBB:
11403 // PHI varPhi, varEnd, varLoop
11404 // PHI srcPhi, src, srcLoop
11405 // PHI destPhi, dst, destLoop
11406 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11407 // [destLoop] = STR_POST(scratch, destPhi, UnitSize)
11408 // subs varLoop, varPhi, #UnitSize
11409 // bne loopMBB
11410 // fallthrough --> exitMBB
11411 // exitMBB:
11412 // epilogue to handle left-over bytes
11413 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11414 // [destOut] = STRB_POST(scratch, destLoop, 1)
11415 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
11416 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
11417 MF->insert(It, loopMBB);
11418 MF->insert(It, exitMBB);
11419
11420 // Set the call frame size on entry to the new basic blocks.
11421 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
11422 loopMBB->setCallFrameSize(CallFrameSize);
11423 exitMBB->setCallFrameSize(CallFrameSize);
11424
11425 // Transfer the remainder of BB and its successor edges to exitMBB.
11426 exitMBB->splice(exitMBB->begin(), BB,
11427 std::next(MachineBasicBlock::iterator(MI)), BB->end());
11429
11430 // Load an immediate to varEnd.
11431 Register varEnd = MRI.createVirtualRegister(TRC);
11432 if (Subtarget->useMovt()) {
11433 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11434 varEnd)
11435 .addImm(LoopSize);
11436 } else if (Subtarget->genExecuteOnly()) {
11437 assert(IsThumb && "Non-thumb expected to have used movt");
11438 BuildMI(BB, dl, TII->get(ARM::tMOVi32imm), varEnd).addImm(LoopSize);
11439 } else {
11440 MachineConstantPool *ConstantPool = MF->getConstantPool();
11441 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext());
11442 const Constant *C = ConstantInt::get(Int32Ty, LoopSize);
11443
11444 // MachineConstantPool wants an explicit alignment.
11446 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11447 MachineMemOperand *CPMMO =
11450
11451 if (IsThumb)
11452 BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci))
11453 .addReg(varEnd, RegState::Define)
11456 .addMemOperand(CPMMO);
11457 else
11458 BuildMI(*BB, MI, dl, TII->get(ARM::LDRcp))
11459 .addReg(varEnd, RegState::Define)
11461 .addImm(0)
11463 .addMemOperand(CPMMO);
11464 }
11465 BB->addSuccessor(loopMBB);
11466
11467 // Generate the loop body:
11468 // varPhi = PHI(varLoop, varEnd)
11469 // srcPhi = PHI(srcLoop, src)
11470 // destPhi = PHI(destLoop, dst)
11471 MachineBasicBlock *entryBB = BB;
11472 BB = loopMBB;
11473 Register varLoop = MRI.createVirtualRegister(TRC);
11474 Register varPhi = MRI.createVirtualRegister(TRC);
11475 Register srcLoop = MRI.createVirtualRegister(TRC);
11476 Register srcPhi = MRI.createVirtualRegister(TRC);
11477 Register destLoop = MRI.createVirtualRegister(TRC);
11478 Register destPhi = MRI.createVirtualRegister(TRC);
11479
11480 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), varPhi)
11481 .addReg(varLoop).addMBB(loopMBB)
11482 .addReg(varEnd).addMBB(entryBB);
11483 BuildMI(BB, dl, TII->get(ARM::PHI), srcPhi)
11484 .addReg(srcLoop).addMBB(loopMBB)
11485 .addReg(src).addMBB(entryBB);
11486 BuildMI(BB, dl, TII->get(ARM::PHI), destPhi)
11487 .addReg(destLoop).addMBB(loopMBB)
11488 .addReg(dest).addMBB(entryBB);
11489
11490 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11491 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz)
11492 Register scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11493 emitPostLd(BB, BB->end(), TII, dl, UnitSize, scratch, srcPhi, srcLoop,
11494 IsThumb1, IsThumb2);
11495 emitPostSt(BB, BB->end(), TII, dl, UnitSize, scratch, destPhi, destLoop,
11496 IsThumb1, IsThumb2);
11497
11498 // Decrement loop variable by UnitSize.
11499 if (IsThumb1) {
11500 BuildMI(*BB, BB->end(), dl, TII->get(ARM::tSUBi8), varLoop)
11501 .add(t1CondCodeOp())
11502 .addReg(varPhi)
11503 .addImm(UnitSize)
11505 } else {
11506 MachineInstrBuilder MIB =
11507 BuildMI(*BB, BB->end(), dl,
11508 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11509 MIB.addReg(varPhi)
11510 .addImm(UnitSize)
11512 .add(condCodeOp());
11513 MIB->getOperand(5).setReg(ARM::CPSR);
11514 MIB->getOperand(5).setIsDef(true);
11515 }
11516 BuildMI(*BB, BB->end(), dl,
11517 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11518 .addMBB(loopMBB).addImm(ARMCC::NE).addReg(ARM::CPSR);
11519
11520 // loopMBB can loop back to loopMBB or fall through to exitMBB.
11521 BB->addSuccessor(loopMBB);
11522 BB->addSuccessor(exitMBB);
11523
11524 // Add epilogue to handle BytesLeft.
11525 BB = exitMBB;
11526 auto StartOfExit = exitMBB->begin();
11527
11528 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11529 // [destOut] = STRB_POST(scratch, destLoop, 1)
11530 unsigned srcIn = srcLoop;
11531 unsigned destIn = destLoop;
11532 for (unsigned i = 0; i < BytesLeft; i++) {
11533 Register srcOut = MRI.createVirtualRegister(TRC);
11534 Register destOut = MRI.createVirtualRegister(TRC);
11535 Register scratch = MRI.createVirtualRegister(TRC);
11536 emitPostLd(BB, StartOfExit, TII, dl, 1, scratch, srcIn, srcOut,
11537 IsThumb1, IsThumb2);
11538 emitPostSt(BB, StartOfExit, TII, dl, 1, scratch, destIn, destOut,
11539 IsThumb1, IsThumb2);
11540 srcIn = srcOut;
11541 destIn = destOut;
11542 }
11543
11544 MI.eraseFromParent(); // The instruction is gone now.
11545 return BB;
11546}
11547
11549ARMTargetLowering::EmitLowered__chkstk(MachineInstr &MI,
11550 MachineBasicBlock *MBB) const {
11551 const TargetMachine &TM = getTargetMachine();
11552 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
11553 DebugLoc DL = MI.getDebugLoc();
11554
11555 assert(TM.getTargetTriple().isOSWindows() &&
11556 "__chkstk is only supported on Windows");
11557 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode");
11558
11559 // __chkstk takes the number of words to allocate on the stack in R4, and
11560 // returns the stack adjustment in number of bytes in R4. This will not
11561 // clober any other registers (other than the obvious lr).
11562 //
11563 // Although, technically, IP should be considered a register which may be
11564 // clobbered, the call itself will not touch it. Windows on ARM is a pure
11565 // thumb-2 environment, so there is no interworking required. As a result, we
11566 // do not expect a veneer to be emitted by the linker, clobbering IP.
11567 //
11568 // Each module receives its own copy of __chkstk, so no import thunk is
11569 // required, again, ensuring that IP is not clobbered.
11570 //
11571 // Finally, although some linkers may theoretically provide a trampoline for
11572 // out of range calls (which is quite common due to a 32M range limitation of
11573 // branches for Thumb), we can generate the long-call version via
11574 // -mcmodel=large, alleviating the need for the trampoline which may clobber
11575 // IP.
11576
11577 RTLIB::LibcallImpl ChkStkLibcall = getLibcallImpl(RTLIB::STACK_PROBE);
11578 if (ChkStkLibcall == RTLIB::Unsupported)
11579 reportFatalUsageError("no available implementation of __chkstk");
11580
11581 const char *ChkStk = getLibcallImplName(ChkStkLibcall).data();
11582 switch (TM.getCodeModel()) {
11583 case CodeModel::Tiny:
11584 llvm_unreachable("Tiny code model not available on ARM.");
11585 case CodeModel::Small:
11586 case CodeModel::Medium:
11587 case CodeModel::Kernel:
11588 BuildMI(*MBB, MI, DL, TII.get(ARM::tBL))
11590 .addExternalSymbol(ChkStk)
11591 .setOperandDead(3) // implicit-def $lr
11594 .addReg(ARM::R12,
11596 .addReg(ARM::CPSR,
11598 break;
11599 case CodeModel::Large: {
11600 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
11601 Register Reg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11602
11603 BuildMI(*MBB, MI, DL, TII.get(ARM::t2MOVi32imm), Reg)
11604 .addExternalSymbol(ChkStk);
11607 .addReg(Reg)
11608 .setOperandDead(3) // implicit-def $lr
11611 .addReg(ARM::R12,
11613 .addReg(ARM::CPSR,
11615 break;
11616 }
11617 }
11618
11619 BuildMI(*MBB, MI, DL, TII.get(ARM::t2SUBrr), ARM::SP)
11620 .addReg(ARM::SP, RegState::Kill)
11621 .addReg(ARM::R4, RegState::Kill)
11624 .add(condCodeOp());
11625
11626 MI.eraseFromParent();
11627 return MBB;
11628}
11629
11631ARMTargetLowering::EmitLowered__dbzchk(MachineInstr &MI,
11632 MachineBasicBlock *MBB) const {
11633 DebugLoc DL = MI.getDebugLoc();
11634 MachineFunction *MF = MBB->getParent();
11635 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11636
11637 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock();
11638 MF->insert(++MBB->getIterator(), ContBB);
11639 ContBB->splice(ContBB->begin(), MBB,
11640 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
11642 MBB->addSuccessor(ContBB);
11643
11644 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
11645 BuildMI(TrapBB, DL, TII->get(ARM::t__brkdiv0));
11646 MF->push_back(TrapBB);
11647 MBB->addSuccessor(TrapBB);
11648
11649 BuildMI(*MBB, MI, DL, TII->get(ARM::tCMPi8))
11650 .addReg(MI.getOperand(0).getReg())
11651 .addImm(0)
11653 BuildMI(*MBB, MI, DL, TII->get(ARM::t2Bcc))
11654 .addMBB(TrapBB)
11656 .addReg(ARM::CPSR);
11657
11658 MI.eraseFromParent();
11659 return ContBB;
11660}
11661
11662// The CPSR operand of SelectItr might be missing a kill marker
11663// because there were multiple uses of CPSR, and ISel didn't know
11664// which to mark. Figure out whether SelectItr should have had a
11665// kill marker, and set it if it should. Returns the correct kill
11666// marker value.
11669 const TargetRegisterInfo* TRI) {
11670 // Scan forward through BB for a use/def of CPSR.
11671 MachineBasicBlock::iterator miI(std::next(SelectItr));
11672 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
11673 const MachineInstr& mi = *miI;
11674 if (mi.readsRegister(ARM::CPSR, /*TRI=*/nullptr))
11675 return false;
11676 if (mi.definesRegister(ARM::CPSR, /*TRI=*/nullptr))
11677 break; // Should have kill-flag - update below.
11678 }
11679
11680 // If we hit the end of the block, check whether CPSR is live into a
11681 // successor.
11682 if (miI == BB->end()) {
11683 for (MachineBasicBlock *Succ : BB->successors())
11684 if (Succ->isLiveIn(ARM::CPSR))
11685 return false;
11686 }
11687
11688 // We found a def, or hit the end of the basic block and CPSR wasn't live
11689 // out. SelectMI should have a kill flag on CPSR.
11690 SelectItr->addRegisterKilled(ARM::CPSR, TRI);
11691 return true;
11692}
11693
11694/// Adds logic in loop entry MBB to calculate loop iteration count and adds
11695/// t2WhileLoopSetup and t2WhileLoopStart to generate WLS loop
11697 MachineBasicBlock *TpLoopBody,
11698 MachineBasicBlock *TpExit, Register OpSizeReg,
11699 const TargetInstrInfo *TII, DebugLoc Dl,
11701 // Calculates loop iteration count = ceil(n/16) = (n + 15) >> 4.
11702 Register AddDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11703 BuildMI(TpEntry, Dl, TII->get(ARM::t2ADDri), AddDestReg)
11704 .addUse(OpSizeReg)
11705 .addImm(15)
11707 .addReg(0);
11708
11709 Register LsrDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11710 BuildMI(TpEntry, Dl, TII->get(ARM::t2LSRri), LsrDestReg)
11711 .addUse(AddDestReg)
11712 .addImm(4)
11714 .addReg(0);
11715
11716 Register TotalIterationsReg = MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11717 BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11718 .addUse(LsrDestReg);
11719
11720 BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopStart))
11721 .addUse(TotalIterationsReg)
11722 .addMBB(TpExit);
11723
11724 BuildMI(TpEntry, Dl, TII->get(ARM::t2B))
11725 .addMBB(TpLoopBody)
11727
11728 return TotalIterationsReg;
11729}
11730
11731/// Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and
11732/// t2DoLoopEnd. These are used by later passes to generate tail predicated
11733/// loops.
11734static void genTPLoopBody(MachineBasicBlock *TpLoopBody,
11735 MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit,
11736 const TargetInstrInfo *TII, DebugLoc Dl,
11737 MachineRegisterInfo &MRI, Register OpSrcReg,
11738 Register OpDestReg, Register ElementCountReg,
11739 Register TotalIterationsReg, bool IsMemcpy) {
11740 // First insert 4 PHI nodes for: Current pointer to Src (if memcpy), Dest
11741 // array, loop iteration counter, predication counter.
11742
11743 Register SrcPhiReg, CurrSrcReg;
11744 if (IsMemcpy) {
11745 // Current position in the src array
11746 SrcPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11747 CurrSrcReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11748 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), SrcPhiReg)
11749 .addUse(OpSrcReg)
11750 .addMBB(TpEntry)
11751 .addUse(CurrSrcReg)
11752 .addMBB(TpLoopBody);
11753 }
11754
11755 // Current position in the dest array
11756 Register DestPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11757 Register CurrDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11758 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), DestPhiReg)
11759 .addUse(OpDestReg)
11760 .addMBB(TpEntry)
11761 .addUse(CurrDestReg)
11762 .addMBB(TpLoopBody);
11763
11764 // Current loop counter
11765 Register LoopCounterPhiReg = MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11766 Register RemainingLoopIterationsReg =
11767 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11768 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), LoopCounterPhiReg)
11769 .addUse(TotalIterationsReg)
11770 .addMBB(TpEntry)
11771 .addUse(RemainingLoopIterationsReg)
11772 .addMBB(TpLoopBody);
11773
11774 // Predication counter
11775 Register PredCounterPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11776 Register RemainingElementsReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11777 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), PredCounterPhiReg)
11778 .addUse(ElementCountReg)
11779 .addMBB(TpEntry)
11780 .addUse(RemainingElementsReg)
11781 .addMBB(TpLoopBody);
11782
11783 // Pass predication counter to VCTP
11784 Register VccrReg = MRI.createVirtualRegister(&ARM::VCCRRegClass);
11785 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VCTP8), VccrReg)
11786 .addUse(PredCounterPhiReg)
11788 .addReg(0)
11789 .addReg(0);
11790
11791 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2SUBri), RemainingElementsReg)
11792 .addUse(PredCounterPhiReg)
11793 .addImm(16)
11795 .addReg(0);
11796
11797 // VLDRB (only if memcpy) and VSTRB instructions, predicated using VPR
11798 Register SrcValueReg;
11799 if (IsMemcpy) {
11800 SrcValueReg = MRI.createVirtualRegister(&ARM::MQPRRegClass);
11801 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VLDRBU8_post))
11802 .addDef(CurrSrcReg)
11803 .addDef(SrcValueReg)
11804 .addReg(SrcPhiReg)
11805 .addImm(16)
11807 .addUse(VccrReg)
11808 .addReg(0);
11809 } else
11810 SrcValueReg = OpSrcReg;
11811
11812 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VSTRBU8_post))
11813 .addDef(CurrDestReg)
11814 .addUse(SrcValueReg)
11815 .addReg(DestPhiReg)
11816 .addImm(16)
11818 .addUse(VccrReg)
11819 .addReg(0);
11820
11821 // Add the pseudoInstrs for decrementing the loop counter and marking the
11822 // end:t2DoLoopDec and t2DoLoopEnd
11823 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11824 .addUse(LoopCounterPhiReg)
11825 .addImm(1);
11826
11827 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopEnd))
11828 .addUse(RemainingLoopIterationsReg)
11829 .addMBB(TpLoopBody);
11830
11831 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2B))
11832 .addMBB(TpExit)
11834}
11835
11837 // KCFI is supported in all ARM/Thumb modes
11838 return true;
11839}
11840
11844 const TargetInstrInfo *TII) const {
11845 assert(MBBI->isCall() && MBBI->getCFIType() &&
11846 "Invalid call instruction for a KCFI check");
11847
11848 MachineOperand *TargetOp = nullptr;
11849 switch (MBBI->getOpcode()) {
11850 // ARM mode opcodes
11851 case ARM::BLX:
11852 case ARM::BLX_pred:
11853 case ARM::BLX_noip:
11854 case ARM::BLX_pred_noip:
11855 case ARM::BX_CALL:
11856 TargetOp = &MBBI->getOperand(0);
11857 break;
11858 case ARM::TCRETURNri:
11859 case ARM::TCRETURNrinotr12:
11860 case ARM::TAILJMPr:
11861 case ARM::TAILJMPr4:
11862 TargetOp = &MBBI->getOperand(0);
11863 break;
11864 // Thumb mode opcodes (Thumb1 and Thumb2)
11865 // Note: Most Thumb call instructions have predicate operands before the
11866 // target register Format: tBLXr pred, predreg, target_register, ...
11867 case ARM::tBLXr: // Thumb1/Thumb2: BLX register (requires V5T)
11868 case ARM::tBLXr_noip: // Thumb1/Thumb2: BLX register, no IP clobber
11869 case ARM::tBX_CALL: // Thumb1 only: BX call (push LR, BX)
11870 TargetOp = &MBBI->getOperand(2);
11871 break;
11872 // Tail call instructions don't have predicates, target is operand 0
11873 case ARM::tTAILJMPr: // Thumb1/Thumb2: Tail call via register
11874 TargetOp = &MBBI->getOperand(0);
11875 break;
11876 default:
11877 llvm_unreachable("Unexpected CFI call opcode");
11878 }
11879
11880 assert(TargetOp && TargetOp->isReg() && "Invalid target operand");
11881 TargetOp->setIsRenamable(false);
11882
11883 // Select the appropriate KCFI_CHECK variant based on the instruction set
11884 unsigned KCFICheckOpcode;
11885 if (Subtarget->isThumb()) {
11886 if (Subtarget->isThumb2()) {
11887 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb2;
11888 } else {
11889 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb1;
11890 }
11891 } else {
11892 KCFICheckOpcode = ARM::KCFI_CHECK_ARM;
11893 }
11894
11895 return BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(KCFICheckOpcode))
11896 .addReg(TargetOp->getReg())
11897 .addImm(MBBI->getCFIType())
11898 .getInstr();
11899}
11900
11903 MachineBasicBlock *BB) const {
11904 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11905 DebugLoc dl = MI.getDebugLoc();
11906 bool isThumb2 = Subtarget->isThumb2();
11907 switch (MI.getOpcode()) {
11908 default: {
11909 MI.print(errs());
11910 llvm_unreachable("Unexpected instr type to insert");
11911 }
11912
11913 // Thumb1 post-indexed loads are really just single-register LDMs.
11914 case ARM::tLDR_postidx: {
11915 MachineOperand Def(MI.getOperand(1));
11916 BuildMI(*BB, MI, dl, TII->get(ARM::tLDMIA_UPD))
11917 .add(Def) // Rn_wb
11918 .add(MI.getOperand(2)) // Rn
11919 .add(MI.getOperand(3)) // PredImm
11920 .add(MI.getOperand(4)) // PredReg
11921 .add(MI.getOperand(0)) // Rt
11922 .cloneMemRefs(MI);
11923 MI.eraseFromParent();
11924 return BB;
11925 }
11926
11927 case ARM::MVE_MEMCPYLOOPINST:
11928 case ARM::MVE_MEMSETLOOPINST: {
11929
11930 // Transformation below expands MVE_MEMCPYLOOPINST/MVE_MEMSETLOOPINST Pseudo
11931 // into a Tail Predicated (TP) Loop. It adds the instructions to calculate
11932 // the iteration count =ceil(size_in_bytes/16)) in the TP entry block and
11933 // adds the relevant instructions in the TP loop Body for generation of a
11934 // WLSTP loop.
11935
11936 // Below is relevant portion of the CFG after the transformation.
11937 // The Machine Basic Blocks are shown along with branch conditions (in
11938 // brackets). Note that TP entry/exit MBBs depict the entry/exit of this
11939 // portion of the CFG and may not necessarily be the entry/exit of the
11940 // function.
11941
11942 // (Relevant) CFG after transformation:
11943 // TP entry MBB
11944 // |
11945 // |-----------------|
11946 // (n <= 0) (n > 0)
11947 // | |
11948 // | TP loop Body MBB<--|
11949 // | | |
11950 // \ |___________|
11951 // \ /
11952 // TP exit MBB
11953
11954 MachineFunction *MF = BB->getParent();
11955 MachineFunctionProperties &Properties = MF->getProperties();
11957
11958 Register OpDestReg = MI.getOperand(0).getReg();
11959 Register OpSrcReg = MI.getOperand(1).getReg();
11960 Register OpSizeReg = MI.getOperand(2).getReg();
11961
11962 // Allocate the required MBBs and add to parent function.
11963 MachineBasicBlock *TpEntry = BB;
11964 MachineBasicBlock *TpLoopBody = MF->CreateMachineBasicBlock();
11965 MachineBasicBlock *TpExit;
11966
11967 MF->push_back(TpLoopBody);
11968
11969 // If any instructions are present in the current block after
11970 // MVE_MEMCPYLOOPINST or MVE_MEMSETLOOPINST, split the current block and
11971 // move the instructions into the newly created exit block. If there are no
11972 // instructions add an explicit branch to the FallThrough block and then
11973 // split.
11974 //
11975 // The split is required for two reasons:
11976 // 1) A terminator(t2WhileLoopStart) will be placed at that site.
11977 // 2) Since a TPLoopBody will be added later, any phis in successive blocks
11978 // need to be updated. splitAt() already handles this.
11979 TpExit = BB->splitAt(MI, false);
11980 if (TpExit == BB) {
11981 assert(BB->canFallThrough() && "Exit Block must be Fallthrough of the "
11982 "block containing memcpy/memset Pseudo");
11983 TpExit = BB->getFallThrough();
11984 BuildMI(BB, dl, TII->get(ARM::t2B))
11985 .addMBB(TpExit)
11987 TpExit = BB->splitAt(MI, false);
11988 }
11989
11990 // Add logic for iteration count
11991 Register TotalIterationsReg =
11992 genTPEntry(TpEntry, TpLoopBody, TpExit, OpSizeReg, TII, dl, MRI);
11993
11994 // Add the vectorized (and predicated) loads/store instructions
11995 bool IsMemcpy = MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
11996 genTPLoopBody(TpLoopBody, TpEntry, TpExit, TII, dl, MRI, OpSrcReg,
11997 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
11998
11999 // Required to avoid conflict with the MachineVerifier during testing.
12000 Properties.resetNoPHIs();
12001
12002 // Connect the blocks
12003 TpEntry->addSuccessor(TpLoopBody);
12004 TpLoopBody->addSuccessor(TpLoopBody);
12005 TpLoopBody->addSuccessor(TpExit);
12006
12007 // Reorder for a more natural layout
12008 TpLoopBody->moveAfter(TpEntry);
12009 TpExit->moveAfter(TpLoopBody);
12010
12011 // Finally, remove the memcpy Pseudo Instruction
12012 MI.eraseFromParent();
12013
12014 // Return the exit block as it may contain other instructions requiring a
12015 // custom inserter
12016 return TpExit;
12017 }
12018
12019 // The Thumb2 pre-indexed stores have the same MI operands, they just
12020 // define them differently in the .td files from the isel patterns, so
12021 // they need pseudos.
12022 case ARM::t2STR_preidx:
12023 MI.setDesc(TII->get(ARM::t2STR_PRE));
12024 return BB;
12025 case ARM::t2STRB_preidx:
12026 MI.setDesc(TII->get(ARM::t2STRB_PRE));
12027 return BB;
12028 case ARM::t2STRH_preidx:
12029 MI.setDesc(TII->get(ARM::t2STRH_PRE));
12030 return BB;
12031
12032 case ARM::STRi_preidx:
12033 case ARM::STRBi_preidx: {
12034 unsigned NewOpc = MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12035 : ARM::STRB_PRE_IMM;
12036 // Decode the offset.
12037 unsigned Offset = MI.getOperand(4).getImm();
12038 bool isSub = ARM_AM::getAM2Op(Offset) == ARM_AM::sub;
12040 if (isSub)
12041 Offset = -Offset;
12042
12043 MachineMemOperand *MMO = *MI.memoperands_begin();
12044 BuildMI(*BB, MI, dl, TII->get(NewOpc))
12045 .add(MI.getOperand(0)) // Rn_wb
12046 .add(MI.getOperand(1)) // Rt
12047 .add(MI.getOperand(2)) // Rn
12048 .addImm(Offset) // offset (skip GPR==zero_reg)
12049 .add(MI.getOperand(5)) // pred
12050 .add(MI.getOperand(6))
12051 .addMemOperand(MMO);
12052 MI.eraseFromParent();
12053 return BB;
12054 }
12055 case ARM::STRr_preidx:
12056 case ARM::STRBr_preidx:
12057 case ARM::STRH_preidx: {
12058 unsigned NewOpc;
12059 switch (MI.getOpcode()) {
12060 default: llvm_unreachable("unexpected opcode!");
12061 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break;
12062 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break;
12063 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break;
12064 }
12065 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc));
12066 for (const MachineOperand &MO : MI.operands())
12067 MIB.add(MO);
12068 MI.eraseFromParent();
12069 return BB;
12070 }
12071
12072 case ARM::tMOVCCr_pseudo: {
12073 // To "insert" a SELECT_CC instruction, we actually have to insert the
12074 // diamond control-flow pattern. The incoming instruction knows the
12075 // destination vreg to set, the condition code register to branch on, the
12076 // true/false values to select between, and a branch opcode to use.
12077 const BasicBlock *LLVM_BB = BB->getBasicBlock();
12079
12080 // thisMBB:
12081 // ...
12082 // TrueVal = ...
12083 // cmpTY ccX, r1, r2
12084 // bCC copy1MBB
12085 // fallthrough --> copy0MBB
12086 MachineBasicBlock *thisMBB = BB;
12087 MachineFunction *F = BB->getParent();
12088 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
12089 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
12090 F->insert(It, copy0MBB);
12091 F->insert(It, sinkMBB);
12092
12093 // Set the call frame size on entry to the new basic blocks.
12094 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
12095 copy0MBB->setCallFrameSize(CallFrameSize);
12096 sinkMBB->setCallFrameSize(CallFrameSize);
12097
12098 // Check whether CPSR is live past the tMOVCCr_pseudo.
12099 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
12100 if (!MI.killsRegister(ARM::CPSR, /*TRI=*/nullptr) &&
12101 !checkAndUpdateCPSRKill(MI, thisMBB, TRI)) {
12102 copy0MBB->addLiveIn(ARM::CPSR);
12103 sinkMBB->addLiveIn(ARM::CPSR);
12104 }
12105
12106 // Transfer the remainder of BB and its successor edges to sinkMBB.
12107 sinkMBB->splice(sinkMBB->begin(), BB,
12108 std::next(MachineBasicBlock::iterator(MI)), BB->end());
12110
12111 BB->addSuccessor(copy0MBB);
12112 BB->addSuccessor(sinkMBB);
12113
12114 BuildMI(BB, dl, TII->get(ARM::tBcc))
12115 .addMBB(sinkMBB)
12116 .addImm(MI.getOperand(3).getImm())
12117 .addReg(MI.getOperand(4).getReg());
12118
12119 // copy0MBB:
12120 // %FalseValue = ...
12121 // # fallthrough to sinkMBB
12122 BB = copy0MBB;
12123
12124 // Update machine-CFG edges
12125 BB->addSuccessor(sinkMBB);
12126
12127 // sinkMBB:
12128 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
12129 // ...
12130 BB = sinkMBB;
12131 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), MI.getOperand(0).getReg())
12132 .addReg(MI.getOperand(1).getReg())
12133 .addMBB(copy0MBB)
12134 .addReg(MI.getOperand(2).getReg())
12135 .addMBB(thisMBB);
12136
12137 MI.eraseFromParent(); // The pseudo instruction is gone now.
12138 return BB;
12139 }
12140
12141 case ARM::BCCi64:
12142 case ARM::BCCZi64: {
12143 // If there is an unconditional branch to the other successor, remove it.
12144 BB->erase(std::next(MachineBasicBlock::iterator(MI)), BB->end());
12145
12146 // Compare both parts that make up the double comparison separately for
12147 // equality.
12148 bool RHSisZero = MI.getOpcode() == ARM::BCCZi64;
12149
12150 Register LHS1 = MI.getOperand(1).getReg();
12151 Register LHS2 = MI.getOperand(2).getReg();
12152 if (RHSisZero) {
12153 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12154 .addReg(LHS1)
12155 .addImm(0)
12157 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12158 .addReg(LHS2).addImm(0)
12159 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
12160 } else {
12161 Register RHS1 = MI.getOperand(3).getReg();
12162 Register RHS2 = MI.getOperand(4).getReg();
12163 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12164 .addReg(LHS1)
12165 .addReg(RHS1)
12167 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12168 .addReg(LHS2).addReg(RHS2)
12169 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
12170 }
12171
12172 MachineBasicBlock *destMBB = MI.getOperand(RHSisZero ? 3 : 5).getMBB();
12173 MachineBasicBlock *exitMBB = OtherSucc(BB, destMBB);
12174 if (MI.getOperand(0).getImm() == ARMCC::NE)
12175 std::swap(destMBB, exitMBB);
12176
12177 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12178 .addMBB(destMBB).addImm(ARMCC::EQ).addReg(ARM::CPSR);
12179 if (isThumb2)
12180 BuildMI(BB, dl, TII->get(ARM::t2B))
12181 .addMBB(exitMBB)
12183 else
12184 BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB);
12185
12186 MI.eraseFromParent(); // The pseudo instruction is gone now.
12187 return BB;
12188 }
12189
12190 case ARM::Int_eh_sjlj_setjmp:
12191 case ARM::Int_eh_sjlj_setjmp_nofp:
12192 case ARM::tInt_eh_sjlj_setjmp:
12193 case ARM::t2Int_eh_sjlj_setjmp:
12194 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12195 return BB;
12196
12197 case ARM::Int_eh_sjlj_setup_dispatch:
12198 EmitSjLjDispatchBlock(MI, BB);
12199 return BB;
12200 case ARM::COPY_STRUCT_BYVAL_I32:
12201 ++NumLoopByVals;
12202 return EmitStructByval(MI, BB);
12203 case ARM::WIN__CHKSTK:
12204 return EmitLowered__chkstk(MI, BB);
12205 case ARM::WIN__DBZCHK:
12206 return EmitLowered__dbzchk(MI, BB);
12207 }
12208}
12209
12210/// Attaches vregs to MEMCPY that it will use as scratch registers
12211/// when it is expanded into LDM/STM. This is done as a post-isel lowering
12212/// instead of as a custom inserter because we need the use list from the SDNode.
12214 MachineInstr &MI, const SDNode *Node) {
12215 bool isThumb1 = Subtarget->isThumb1Only();
12216
12217 MachineFunction *MF = MI.getParent()->getParent();
12219 MachineInstrBuilder MIB(*MF, MI);
12220
12221 // If the new dst/src is unused mark it as dead.
12222 if (!Node->hasAnyUseOfValue(0)) {
12223 MI.getOperand(0).setIsDead(true);
12224 }
12225 if (!Node->hasAnyUseOfValue(1)) {
12226 MI.getOperand(1).setIsDead(true);
12227 }
12228
12229 // The MEMCPY both defines and kills the scratch registers.
12230 for (unsigned I = 0; I != MI.getOperand(4).getImm(); ++I) {
12231 Register TmpReg = MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12232 : &ARM::GPRRegClass);
12234 }
12235}
12236
12238 SDNode *Node) const {
12239 if (MI.getOpcode() == ARM::MEMCPY) {
12240 attachMEMCPYScratchRegs(Subtarget, MI, Node);
12241 return;
12242 }
12243
12244 const MCInstrDesc *MCID = &MI.getDesc();
12245 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB,
12246 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional
12247 // operand is still set to noreg. If needed, set the optional operand's
12248 // register to CPSR, and remove the redundant implicit def.
12249 //
12250 // e.g. ADCS (..., implicit-def CPSR) -> ADC (... opt:def CPSR).
12251
12252 // Rename pseudo opcodes.
12253 unsigned NewOpc = convertAddSubFlagsOpcode(MI.getOpcode());
12254 unsigned ccOutIdx;
12255 if (NewOpc) {
12256 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo();
12257 MCID = &TII->get(NewOpc);
12258
12259 assert(MCID->getNumOperands() ==
12260 MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize()
12261 && "converted opcode should be the same except for cc_out"
12262 " (and, on Thumb1, pred)");
12263
12264 MI.setDesc(*MCID);
12265
12266 // Add the optional cc_out operand
12267 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/true));
12268
12269 // On Thumb1, move all input operands to the end, then add the predicate
12270 if (Subtarget->isThumb1Only()) {
12271 for (unsigned c = MCID->getNumOperands() - 4; c--;) {
12272 MI.addOperand(MI.getOperand(1));
12273 MI.removeOperand(1);
12274 }
12275
12276 // Restore the ties
12277 for (unsigned i = MI.getNumOperands(); i--;) {
12278 const MachineOperand& op = MI.getOperand(i);
12279 if (op.isReg() && op.isUse()) {
12280 int DefIdx = MCID->getOperandConstraint(i, MCOI::TIED_TO);
12281 if (DefIdx != -1)
12282 MI.tieOperands(DefIdx, i);
12283 }
12284 }
12285
12287 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/false));
12288 ccOutIdx = 1;
12289 } else
12290 ccOutIdx = MCID->getNumOperands() - 1;
12291 } else
12292 ccOutIdx = MCID->getNumOperands() - 1;
12293
12294 // Any ARM instruction that sets the 's' bit should specify an optional
12295 // "cc_out" operand in the last operand position.
12296 if (!MI.hasOptionalDef() || !MCID->operands()[ccOutIdx].isOptionalDef()) {
12297 assert(!NewOpc && "Optional cc_out operand required");
12298 return;
12299 }
12300 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it
12301 // since we already have an optional CPSR def.
12302 bool definesCPSR = false;
12303 bool deadCPSR = false;
12304 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e;
12305 ++i) {
12306 const MachineOperand &MO = MI.getOperand(i);
12307 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) {
12308 definesCPSR = true;
12309 if (MO.isDead())
12310 deadCPSR = true;
12311 MI.removeOperand(i);
12312 break;
12313 }
12314 }
12315 if (!definesCPSR) {
12316 assert(!NewOpc && "Optional cc_out operand required");
12317 return;
12318 }
12319 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag");
12320 if (deadCPSR) {
12321 assert(!MI.getOperand(ccOutIdx).getReg() &&
12322 "expect uninitialized optional cc_out operand");
12323 // Thumb1 instructions must have the S bit even if the CPSR is dead.
12324 if (!Subtarget->isThumb1Only())
12325 return;
12326 }
12327
12328 // If this instruction was defined with an optional CPSR def and its dag node
12329 // had a live implicit CPSR def, then activate the optional CPSR def.
12330 MachineOperand &MO = MI.getOperand(ccOutIdx);
12331 MO.setReg(ARM::CPSR);
12332 MO.setIsDef(true);
12333 MO.setIsDead(deadCPSR);
12334}
12335
12336//===----------------------------------------------------------------------===//
12337// ARM Optimization Hooks
12338//===----------------------------------------------------------------------===//
12339
12340// Helper function that checks if N is a null or all ones constant.
12341static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
12343}
12344
12345// Return true if N is conditionally 0 or all ones.
12346// Detects these expressions where cc is an i1 value:
12347//
12348// (select cc 0, y) [AllOnes=0]
12349// (select cc y, 0) [AllOnes=0]
12350// (zext cc) [AllOnes=0]
12351// (sext cc) [AllOnes=0/1]
12352// (select cc -1, y) [AllOnes=1]
12353// (select cc y, -1) [AllOnes=1]
12354//
12355// Invert is set when N is the null/all ones constant when CC is false.
12356// OtherOp is set to the alternative value of N.
12358 SDValue &CC, bool &Invert,
12359 SDValue &OtherOp,
12360 SelectionDAG &DAG) {
12361 switch (N->getOpcode()) {
12362 default: return false;
12363 case ISD::SELECT: {
12364 CC = N->getOperand(0);
12365 SDValue N1 = N->getOperand(1);
12366 SDValue N2 = N->getOperand(2);
12367 if (isZeroOrAllOnes(N1, AllOnes)) {
12368 Invert = false;
12369 OtherOp = N2;
12370 return true;
12371 }
12372 if (isZeroOrAllOnes(N2, AllOnes)) {
12373 Invert = true;
12374 OtherOp = N1;
12375 return true;
12376 }
12377 return false;
12378 }
12379 case ISD::ZERO_EXTEND:
12380 // (zext cc) can never be the all ones value.
12381 if (AllOnes)
12382 return false;
12383 [[fallthrough]];
12384 case ISD::SIGN_EXTEND: {
12385 SDLoc dl(N);
12386 EVT VT = N->getValueType(0);
12387 CC = N->getOperand(0);
12388 if (CC.getValueType() != MVT::i1 || CC.getOpcode() != ISD::SETCC)
12389 return false;
12390 Invert = !AllOnes;
12391 if (AllOnes)
12392 // When looking for an AllOnes constant, N is an sext, and the 'other'
12393 // value is 0.
12394 OtherOp = DAG.getConstant(0, dl, VT);
12395 else if (N->getOpcode() == ISD::ZERO_EXTEND)
12396 // When looking for a 0 constant, N can be zext or sext.
12397 OtherOp = DAG.getConstant(1, dl, VT);
12398 else
12399 OtherOp = DAG.getAllOnesConstant(dl, VT);
12400 return true;
12401 }
12402 }
12403}
12404
12405// Combine a constant select operand into its use:
12406//
12407// (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
12408// (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
12409// (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
12410// (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
12411// (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
12412//
12413// The transform is rejected if the select doesn't have a constant operand that
12414// is null, or all ones when AllOnes is set.
12415//
12416// Also recognize sext/zext from i1:
12417//
12418// (add (zext cc), x) -> (select cc (add x, 1), x)
12419// (add (sext cc), x) -> (select cc (add x, -1), x)
12420//
12421// These transformations eventually create predicated instructions.
12422//
12423// @param N The node to transform.
12424// @param Slct The N operand that is a select.
12425// @param OtherOp The other N operand (x above).
12426// @param DCI Context.
12427// @param AllOnes Require the select constant to be all ones instead of null.
12428// @returns The new node, or SDValue() on failure.
12429static
12432 bool AllOnes = false) {
12433 SelectionDAG &DAG = DCI.DAG;
12434 EVT VT = N->getValueType(0);
12435 SDValue NonConstantVal;
12436 SDValue CCOp;
12437 bool SwapSelectOps;
12438 if (!isConditionalZeroOrAllOnes(Slct.getNode(), AllOnes, CCOp, SwapSelectOps,
12439 NonConstantVal, DAG))
12440 return SDValue();
12441
12442 // Slct is now know to be the desired identity constant when CC is true.
12443 SDValue TrueVal = OtherOp;
12444 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
12445 OtherOp, NonConstantVal);
12446 // Unless SwapSelectOps says CC should be false.
12447 if (SwapSelectOps)
12448 std::swap(TrueVal, FalseVal);
12449
12450 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
12451 CCOp, TrueVal, FalseVal);
12452}
12453
12454// Attempt combineSelectAndUse on each operand of a commutative operator N.
12455static
12458 SDValue N0 = N->getOperand(0);
12459 SDValue N1 = N->getOperand(1);
12460 if (N0.getNode()->hasOneUse())
12461 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes))
12462 return Result;
12463 if (N1.getNode()->hasOneUse())
12464 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes))
12465 return Result;
12466 return SDValue();
12467}
12468
12470 // VUZP shuffle node.
12471 if (N->getOpcode() == ARMISD::VUZP)
12472 return true;
12473
12474 // "VUZP" on i32 is an alias for VTRN.
12475 if (N->getOpcode() == ARMISD::VTRN && N->getValueType(0) == MVT::v2i32)
12476 return true;
12477
12478 return false;
12479}
12480
12483 const ARMSubtarget *Subtarget) {
12484 // Look for ADD(VUZP.0, VUZP.1).
12485 if (!IsVUZPShuffleNode(N0.getNode()) || N0.getNode() != N1.getNode() ||
12486 N0 == N1)
12487 return SDValue();
12488
12489 // Make sure the ADD is a 64-bit add; there is no 128-bit VPADD.
12490 if (!N->getValueType(0).is64BitVector())
12491 return SDValue();
12492
12493 // Generate vpadd.
12494 SelectionDAG &DAG = DCI.DAG;
12496 SDLoc dl(N);
12497 SDNode *Unzip = N0.getNode();
12498 EVT VT = N->getValueType(0);
12499
12501 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpadd, dl,
12502 TLI.getPointerTy(DAG.getDataLayout())));
12503 Ops.push_back(Unzip->getOperand(0));
12504 Ops.push_back(Unzip->getOperand(1));
12505
12506 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops);
12507}
12508
12511 const ARMSubtarget *Subtarget) {
12512 // Check for two extended operands.
12513 if (!(N0.getOpcode() == ISD::SIGN_EXTEND &&
12514 N1.getOpcode() == ISD::SIGN_EXTEND) &&
12515 !(N0.getOpcode() == ISD::ZERO_EXTEND &&
12516 N1.getOpcode() == ISD::ZERO_EXTEND))
12517 return SDValue();
12518
12519 SDValue N00 = N0.getOperand(0);
12520 SDValue N10 = N1.getOperand(0);
12521
12522 // Look for ADD(SEXT(VUZP.0), SEXT(VUZP.1))
12523 if (!IsVUZPShuffleNode(N00.getNode()) || N00.getNode() != N10.getNode() ||
12524 N00 == N10)
12525 return SDValue();
12526
12527 // We only recognize Q register paddl here; this can't be reached until
12528 // after type legalization.
12529 if (!N00.getValueType().is64BitVector() ||
12531 return SDValue();
12532
12533 // Generate vpaddl.
12534 SelectionDAG &DAG = DCI.DAG;
12536 SDLoc dl(N);
12537 EVT VT = N->getValueType(0);
12538
12540 // Form vpaddl.sN or vpaddl.uN depending on the kind of extension.
12541 unsigned Opcode;
12542 if (N0.getOpcode() == ISD::SIGN_EXTEND)
12543 Opcode = Intrinsic::arm_neon_vpaddls;
12544 else
12545 Opcode = Intrinsic::arm_neon_vpaddlu;
12546 Ops.push_back(DAG.getConstant(Opcode, dl,
12547 TLI.getPointerTy(DAG.getDataLayout())));
12549 unsigned NumElts = VT.getVectorNumElements();
12550 EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), ElemTy, NumElts * 2);
12551 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT,
12552 N00.getOperand(0), N00.getOperand(1));
12553 Ops.push_back(Concat);
12554
12555 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops);
12556}
12557
12558// FIXME: This function shouldn't be necessary; if we lower BUILD_VECTOR in
12559// an appropriate manner, we end up with ADD(VUZP(ZEXT(N))), which is
12560// much easier to match.
12561static SDValue
12564 const ARMSubtarget *Subtarget) {
12565 // Only perform optimization if after legalize, and if NEON is available. We
12566 // also expected both operands to be BUILD_VECTORs.
12567 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON()
12568 || N0.getOpcode() != ISD::BUILD_VECTOR
12569 || N1.getOpcode() != ISD::BUILD_VECTOR)
12570 return SDValue();
12571
12572 // Check output type since VPADDL operand elements can only be 8, 16, or 32.
12573 EVT VT = N->getValueType(0);
12574 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64)
12575 return SDValue();
12576
12577 // Check that the vector operands are of the right form.
12578 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR
12579 // operands, where N is the size of the formed vector.
12580 // Each EXTRACT_VECTOR should have the same input vector and odd or even
12581 // index such that we have a pair wise add pattern.
12582
12583 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing.
12585 return SDValue();
12586 SDValue Vec = N0->getOperand(0)->getOperand(0);
12587 SDNode *V = Vec.getNode();
12588 unsigned nextIndex = 0;
12589
12590 // For each operands to the ADD which are BUILD_VECTORs,
12591 // check to see if each of their operands are an EXTRACT_VECTOR with
12592 // the same vector and appropriate index.
12593 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) {
12596
12597 SDValue ExtVec0 = N0->getOperand(i);
12598 SDValue ExtVec1 = N1->getOperand(i);
12599
12600 // First operand is the vector, verify its the same.
12601 if (V != ExtVec0->getOperand(0).getNode() ||
12602 V != ExtVec1->getOperand(0).getNode())
12603 return SDValue();
12604
12605 // Second is the constant, verify its correct.
12608
12609 // For the constant, we want to see all the even or all the odd.
12610 if (!C0 || !C1 || C0->getZExtValue() != nextIndex
12611 || C1->getZExtValue() != nextIndex+1)
12612 return SDValue();
12613
12614 // Increment index.
12615 nextIndex+=2;
12616 } else
12617 return SDValue();
12618 }
12619
12620 // Don't generate vpaddl+vmovn; we'll match it to vpadd later. Also make sure
12621 // we're using the entire input vector, otherwise there's a size/legality
12622 // mismatch somewhere.
12623 if (nextIndex != Vec.getValueType().getVectorNumElements() ||
12625 return SDValue();
12626
12627 // Create VPADDL node.
12628 SelectionDAG &DAG = DCI.DAG;
12630
12631 SDLoc dl(N);
12632
12633 // Build operand list.
12635 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls, dl,
12636 TLI.getPointerTy(DAG.getDataLayout())));
12637
12638 // Input is the vector.
12639 Ops.push_back(Vec);
12640
12641 // Get widened type and narrowed type.
12642 MVT widenType;
12643 unsigned numElem = VT.getVectorNumElements();
12644
12645 EVT inputLaneType = Vec.getValueType().getVectorElementType();
12646 switch (inputLaneType.getSimpleVT().SimpleTy) {
12647 case MVT::i8: widenType = MVT::getVectorVT(MVT::i16, numElem); break;
12648 case MVT::i16: widenType = MVT::getVectorVT(MVT::i32, numElem); break;
12649 case MVT::i32: widenType = MVT::getVectorVT(MVT::i64, numElem); break;
12650 default:
12651 llvm_unreachable("Invalid vector element type for padd optimization.");
12652 }
12653
12654 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops);
12655 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE;
12656 return DAG.getNode(ExtOp, dl, VT, tmp);
12657}
12658
12660 if (V->getOpcode() == ISD::UMUL_LOHI ||
12661 V->getOpcode() == ISD::SMUL_LOHI)
12662 return V;
12663 return SDValue();
12664}
12665
12666static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode,
12668 const ARMSubtarget *Subtarget) {
12669 if (!Subtarget->hasBaseDSP())
12670 return SDValue();
12671
12672 // SMLALBB, SMLALBT, SMLALTB, SMLALTT multiply two 16-bit values and
12673 // accumulates the product into a 64-bit value. The 16-bit values will
12674 // be sign extended somehow or SRA'd into 32-bit values
12675 // (addc (adde (mul 16bit, 16bit), lo), hi)
12676 SDValue Mul = AddcNode->getOperand(0);
12677 SDValue Lo = AddcNode->getOperand(1);
12678 if (Mul.getOpcode() != ISD::MUL) {
12679 Lo = AddcNode->getOperand(0);
12680 Mul = AddcNode->getOperand(1);
12681 if (Mul.getOpcode() != ISD::MUL)
12682 return SDValue();
12683 }
12684
12685 SDValue SRA = AddeNode->getOperand(0);
12686 SDValue Hi = AddeNode->getOperand(1);
12687 if (SRA.getOpcode() != ISD::SRA) {
12688 SRA = AddeNode->getOperand(1);
12689 Hi = AddeNode->getOperand(0);
12690 if (SRA.getOpcode() != ISD::SRA)
12691 return SDValue();
12692 }
12693 if (auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12694 if (Const->getZExtValue() != 31)
12695 return SDValue();
12696 } else
12697 return SDValue();
12698
12699 if (SRA.getOperand(0) != Mul)
12700 return SDValue();
12701
12702 SelectionDAG &DAG = DCI.DAG;
12703 SDLoc dl(AddcNode);
12704 unsigned Opcode = 0;
12705 SDValue Op0;
12706 SDValue Op1;
12707
12708 if (isS16(Mul.getOperand(0), DAG) && isS16(Mul.getOperand(1), DAG)) {
12709 Opcode = ARMISD::SMLALBB;
12710 Op0 = Mul.getOperand(0);
12711 Op1 = Mul.getOperand(1);
12712 } else if (isS16(Mul.getOperand(0), DAG) && isSRA16(Mul.getOperand(1))) {
12713 Opcode = ARMISD::SMLALBT;
12714 Op0 = Mul.getOperand(0);
12715 Op1 = Mul.getOperand(1).getOperand(0);
12716 } else if (isSRA16(Mul.getOperand(0)) && isS16(Mul.getOperand(1), DAG)) {
12717 Opcode = ARMISD::SMLALTB;
12718 Op0 = Mul.getOperand(0).getOperand(0);
12719 Op1 = Mul.getOperand(1);
12720 } else if (isSRA16(Mul.getOperand(0)) && isSRA16(Mul.getOperand(1))) {
12721 Opcode = ARMISD::SMLALTT;
12722 Op0 = Mul->getOperand(0).getOperand(0);
12723 Op1 = Mul->getOperand(1).getOperand(0);
12724 }
12725
12726 if (!Op0 || !Op1)
12727 return SDValue();
12728
12729 SDValue SMLAL = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
12730 Op0, Op1, Lo, Hi);
12731 // Replace the ADDs' nodes uses by the MLA node's values.
12732 SDValue HiMLALResult(SMLAL.getNode(), 1);
12733 SDValue LoMLALResult(SMLAL.getNode(), 0);
12734
12735 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult);
12736 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult);
12737 DCI.recursivelyDeleteUnusedNodes(AddeNode);
12738
12739 // Return original node to notify the driver to stop replacing.
12740 return SDValue(AddeNode, 0);
12741}
12742
12745 const ARMSubtarget *Subtarget) {
12746 // Look for multiply add opportunities.
12747 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where
12748 // each add nodes consumes a value from ISD::UMUL_LOHI and there is
12749 // a glue link from the first add to the second add.
12750 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
12751 // a S/UMLAL instruction.
12752 // UMUL_LOHI
12753 // / :lo \ :hi
12754 // V \ [no multiline comment]
12755 // loAdd -> ADDC |
12756 // \ :carry /
12757 // V V
12758 // ADDE <- hiAdd
12759 //
12760 // In the special case where only the higher part of a signed result is used
12761 // and the add to the low part of the result of ISD::UMUL_LOHI adds or subtracts
12762 // a constant with the exact value of 0x80000000, we recognize we are dealing
12763 // with a "rounded multiply and add" (or subtract) and transform it into
12764 // either a ARMISD::SMMLAR or ARMISD::SMMLSR respectively.
12765
12766 assert((AddeSubeNode->getOpcode() == ARMISD::ADDE ||
12767 AddeSubeNode->getOpcode() == ARMISD::SUBE) &&
12768 "Expect an ADDE or SUBE");
12769
12770 assert(AddeSubeNode->getNumOperands() == 3 &&
12771 AddeSubeNode->getOperand(2).getValueType() == MVT::i32 &&
12772 "ADDE node has the wrong inputs");
12773
12774 if (AddeSubeNode->hasAnyUseOfValue(1))
12775 return SDValue();
12776
12777 // Check that we are chained to the right ADDC or SUBC node.
12778 SDNode *AddcSubcNode = AddeSubeNode->getOperand(2).getNode();
12779 if ((AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12780 AddcSubcNode->getOpcode() != ARMISD::ADDC) ||
12781 (AddeSubeNode->getOpcode() == ARMISD::SUBE &&
12782 AddcSubcNode->getOpcode() != ARMISD::SUBC))
12783 return SDValue();
12784
12785 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(0);
12786 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(1);
12787
12788 // Check if the two operands are from the same mul_lohi node.
12789 if (AddcSubcOp0.getNode() == AddcSubcOp1.getNode())
12790 return SDValue();
12791
12792 assert(AddcSubcNode->getNumValues() == 2 &&
12793 AddcSubcNode->getValueType(0) == MVT::i32 &&
12794 "Expect ADDC with two result values. First: i32");
12795
12796 // Check that the ADDC adds the low result of the S/UMUL_LOHI. If not, it
12797 // maybe a SMLAL which multiplies two 16-bit values.
12798 if (AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12799 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI &&
12800 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI &&
12801 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI &&
12802 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI)
12803 return AddCombineTo64BitSMLAL16(AddcSubcNode, AddeSubeNode, DCI, Subtarget);
12804
12805 // Check for the triangle shape.
12806 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(0);
12807 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(1);
12808
12809 // Make sure that the ADDE/SUBE operands are not coming from the same node.
12810 if (AddeSubeOp0.getNode() == AddeSubeOp1.getNode())
12811 return SDValue();
12812
12813 // Find the MUL_LOHI node walking up ADDE/SUBE's operands.
12814 bool IsLeftOperandMUL = false;
12815 SDValue MULOp = findMUL_LOHI(AddeSubeOp0);
12816 if (MULOp == SDValue())
12817 MULOp = findMUL_LOHI(AddeSubeOp1);
12818 else
12819 IsLeftOperandMUL = true;
12820 if (MULOp == SDValue())
12821 return SDValue();
12822
12823 // Figure out the right opcode.
12824 unsigned Opc = MULOp->getOpcode();
12825 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL;
12826
12827 // Figure out the high and low input values to the MLAL node.
12828 SDValue *HiAddSub = nullptr;
12829 SDValue *LoMul = nullptr;
12830 SDValue *LowAddSub = nullptr;
12831
12832 // Ensure that ADDE/SUBE is from high result of ISD::xMUL_LOHI.
12833 if ((AddeSubeOp0 != MULOp.getValue(1)) && (AddeSubeOp1 != MULOp.getValue(1)))
12834 return SDValue();
12835
12836 if (IsLeftOperandMUL)
12837 HiAddSub = &AddeSubeOp1;
12838 else
12839 HiAddSub = &AddeSubeOp0;
12840
12841 // Ensure that LoMul and LowAddSub are taken from correct ISD::SMUL_LOHI node
12842 // whose low result is fed to the ADDC/SUBC we are checking.
12843
12844 if (AddcSubcOp0 == MULOp.getValue(0)) {
12845 LoMul = &AddcSubcOp0;
12846 LowAddSub = &AddcSubcOp1;
12847 }
12848 if (AddcSubcOp1 == MULOp.getValue(0)) {
12849 LoMul = &AddcSubcOp1;
12850 LowAddSub = &AddcSubcOp0;
12851 }
12852
12853 if (!LoMul)
12854 return SDValue();
12855
12856 // If HiAddSub is the same node as ADDC/SUBC or is a predecessor of ADDC/SUBC
12857 // the replacement below will create a cycle.
12858 if (AddcSubcNode == HiAddSub->getNode() ||
12859 AddcSubcNode->isPredecessorOf(HiAddSub->getNode()))
12860 return SDValue();
12861
12862 // Create the merged node.
12863 SelectionDAG &DAG = DCI.DAG;
12864
12865 // Start building operand list.
12867 Ops.push_back(LoMul->getOperand(0));
12868 Ops.push_back(LoMul->getOperand(1));
12869
12870 // Check whether we can use SMMLAR, SMMLSR or SMMULR instead. For this to be
12871 // the case, we must be doing signed multiplication and only use the higher
12872 // part of the result of the MLAL, furthermore the LowAddSub must be a constant
12873 // addition or subtraction with the value of 0x800000.
12874 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->useMulOps() &&
12875 FinalOpc == ARMISD::SMLAL &&
12876 LowAddSub->getNode()->getOpcode() == ISD::Constant &&
12877 static_cast<ConstantSDNode *>(LowAddSub->getNode())->getZExtValue() ==
12878 0x80000000) {
12879 Ops.push_back(*HiAddSub);
12880 if (AddcSubcNode->getOpcode() == ARMISD::SUBC) {
12881 FinalOpc = ARMISD::SMMLSR;
12882 } else {
12883 FinalOpc = ARMISD::SMMLAR;
12884 }
12885 SDValue NewNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), MVT::i32, Ops);
12886 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), NewNode);
12887
12888 return SDValue(AddeSubeNode, 0);
12889 } else if (AddcSubcNode->getOpcode() == ARMISD::SUBC)
12890 // SMMLS is generated during instruction selection and the rest of this
12891 // function can not handle the case where AddcSubcNode is a SUBC.
12892 return SDValue();
12893
12894 // Finish building the operand list for {U/S}MLAL
12895 Ops.push_back(*LowAddSub);
12896 Ops.push_back(*HiAddSub);
12897
12898 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode),
12899 DAG.getVTList(MVT::i32, MVT::i32), Ops);
12900
12901 // Replace the ADDs' nodes uses by the MLA node's values.
12902 SDValue HiMLALResult(MLALNode.getNode(), 1);
12903 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), HiMLALResult);
12904
12905 SDValue LoMLALResult(MLALNode.getNode(), 0);
12906 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcSubcNode, 0), LoMLALResult);
12907
12908 // Return original node to notify the driver to stop replacing.
12909 return SDValue(AddeSubeNode, 0);
12910}
12911
12914 const ARMSubtarget *Subtarget) {
12915 // UMAAL is similar to UMLAL except that it adds two unsigned values.
12916 // While trying to combine for the other MLAL nodes, first search for the
12917 // chance to use UMAAL. Check if Addc uses a node which has already
12918 // been combined into a UMLAL. The other pattern is UMLAL using Addc/Adde
12919 // as the addend, and it's handled in PerformUMLALCombine.
12920
12921 if (AddeNode->hasAnyUseOfValue(1))
12922 return SDValue();
12923
12924 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12925 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget);
12926
12927 // Check that we have a glued ADDC node.
12928 SDNode* AddcNode = AddeNode->getOperand(2).getNode();
12929 if (AddcNode->getOpcode() != ARMISD::ADDC)
12930 return SDValue();
12931
12932 // Find the converted UMAAL or quit if it doesn't exist.
12933 SDNode *UmlalNode = nullptr;
12934 SDValue AddHi;
12935 if (AddcNode->getOperand(0).getOpcode() == ARMISD::UMLAL) {
12936 UmlalNode = AddcNode->getOperand(0).getNode();
12937 AddHi = AddcNode->getOperand(1);
12938 } else if (AddcNode->getOperand(1).getOpcode() == ARMISD::UMLAL) {
12939 UmlalNode = AddcNode->getOperand(1).getNode();
12940 AddHi = AddcNode->getOperand(0);
12941 } else {
12942 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget);
12943 }
12944
12945 // The ADDC should be glued to an ADDE node, which uses the same UMLAL as
12946 // the ADDC as well as Zero.
12947 if (!isNullConstant(UmlalNode->getOperand(3)))
12948 return SDValue();
12949
12950 if ((isNullConstant(AddeNode->getOperand(0)) &&
12951 AddeNode->getOperand(1).getNode() == UmlalNode) ||
12952 (AddeNode->getOperand(0).getNode() == UmlalNode &&
12953 isNullConstant(AddeNode->getOperand(1)))) {
12954 SelectionDAG &DAG = DCI.DAG;
12955 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1),
12956 UmlalNode->getOperand(2), AddHi };
12957 SDValue UMAAL = DAG.getNode(ARMISD::UMAAL, SDLoc(AddcNode),
12958 DAG.getVTList(MVT::i32, MVT::i32), Ops);
12959
12960 // Replace the ADDs' nodes uses by the UMAAL node's values.
12961 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), SDValue(UMAAL.getNode(), 1));
12962 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), SDValue(UMAAL.getNode(), 0));
12963
12964 // Return original node to notify the driver to stop replacing.
12965 return SDValue(AddeNode, 0);
12966 }
12967 return SDValue();
12968}
12969
12971 const ARMSubtarget *Subtarget) {
12972 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12973 return SDValue();
12974
12975 // Check that we have a pair of ADDC and ADDE as operands.
12976 // Both addends of the ADDE must be zero.
12977 SDNode* AddcNode = N->getOperand(2).getNode();
12978 SDNode* AddeNode = N->getOperand(3).getNode();
12979 if ((AddcNode->getOpcode() == ARMISD::ADDC) &&
12980 (AddeNode->getOpcode() == ARMISD::ADDE) &&
12981 isNullConstant(AddeNode->getOperand(0)) &&
12982 isNullConstant(AddeNode->getOperand(1)) &&
12983 (AddeNode->getOperand(2).getNode() == AddcNode))
12984 return DAG.getNode(ARMISD::UMAAL, SDLoc(N),
12985 DAG.getVTList(MVT::i32, MVT::i32),
12986 {N->getOperand(0), N->getOperand(1),
12987 AddcNode->getOperand(0), AddcNode->getOperand(1)});
12988 else
12989 return SDValue();
12990}
12991
12994 const ARMSubtarget *Subtarget) {
12995 SelectionDAG &DAG(DCI.DAG);
12996
12997 if (N->getOpcode() == ARMISD::SUBC && N->hasAnyUseOfValue(1)) {
12998 // (SUBC (ADDE 0, 0, C), 1) -> C
12999 SDValue LHS = N->getOperand(0);
13000 SDValue RHS = N->getOperand(1);
13001 if (LHS->getOpcode() == ARMISD::ADDE &&
13002 isNullConstant(LHS->getOperand(0)) &&
13003 isNullConstant(LHS->getOperand(1)) && isOneConstant(RHS)) {
13004 return DCI.CombineTo(N, SDValue(N, 0), LHS->getOperand(2));
13005 }
13006 }
13007
13008 if (Subtarget->isThumb1Only()) {
13009 SDValue RHS = N->getOperand(1);
13011 int32_t imm = C->getSExtValue();
13012 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13013 SDLoc DL(N);
13014 RHS = DAG.getConstant(-imm, DL, MVT::i32);
13015 unsigned Opcode = (N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC
13016 : ARMISD::ADDC;
13017 return DAG.getNode(Opcode, DL, N->getVTList(), N->getOperand(0), RHS);
13018 }
13019 }
13020 }
13021
13022 return SDValue();
13023}
13024
13027 const ARMSubtarget *Subtarget) {
13028 if (Subtarget->isThumb1Only()) {
13029 SelectionDAG &DAG = DCI.DAG;
13030 SDValue RHS = N->getOperand(1);
13032 int64_t imm = C->getSExtValue();
13033 if (imm < 0) {
13034 SDLoc DL(N);
13035
13036 // The with-carry-in form matches bitwise not instead of the negation.
13037 // Effectively, the inverse interpretation of the carry flag already
13038 // accounts for part of the negation.
13039 RHS = DAG.getConstant(~imm, DL, MVT::i32);
13040
13041 unsigned Opcode = (N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE
13042 : ARMISD::ADDE;
13043 return DAG.getNode(Opcode, DL, N->getVTList(),
13044 N->getOperand(0), RHS, N->getOperand(2));
13045 }
13046 }
13047 } else if (N->getOperand(1)->getOpcode() == ISD::SMUL_LOHI) {
13048 return AddCombineTo64bitMLAL(N, DCI, Subtarget);
13049 }
13050 return SDValue();
13051}
13052
13055 const ARMSubtarget *Subtarget) {
13056 if (!Subtarget->hasMVEIntegerOps())
13057 return SDValue();
13058
13059 SDLoc dl(N);
13060 SDValue SetCC;
13061 SDValue LHS;
13062 SDValue RHS;
13064 SDValue TrueVal;
13065 SDValue FalseVal;
13066
13067 if (N->getOpcode() == ISD::SELECT &&
13068 N->getOperand(0)->getOpcode() == ISD::SETCC) {
13069 SetCC = N->getOperand(0);
13070 LHS = SetCC->getOperand(0);
13071 RHS = SetCC->getOperand(1);
13072 CC = SetCC->getOperandCondCode(2);
13073 TrueVal = N->getOperand(1);
13074 FalseVal = N->getOperand(2);
13075 } else if (N->getOpcode() == ISD::SELECT_CC) {
13076 LHS = N->getOperand(0);
13077 RHS = N->getOperand(1);
13078 CC = N->getOperandCondCode(4);
13079 TrueVal = N->getOperand(2);
13080 FalseVal = N->getOperand(3);
13081 } else {
13082 return SDValue();
13083 }
13084
13085 unsigned int Opcode = 0;
13086 if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMIN ||
13087 FalseVal->getOpcode() == ISD::VECREDUCE_UMIN) &&
13088 (CC == ISD::SETULT || CC == ISD::SETUGT)) {
13089 Opcode = ARMISD::VMINVu;
13090 if (CC == ISD::SETUGT)
13091 std::swap(TrueVal, FalseVal);
13092 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMIN ||
13093 FalseVal->getOpcode() == ISD::VECREDUCE_SMIN) &&
13094 (CC == ISD::SETLT || CC == ISD::SETGT)) {
13095 Opcode = ARMISD::VMINVs;
13096 if (CC == ISD::SETGT)
13097 std::swap(TrueVal, FalseVal);
13098 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMAX ||
13099 FalseVal->getOpcode() == ISD::VECREDUCE_UMAX) &&
13100 (CC == ISD::SETUGT || CC == ISD::SETULT)) {
13101 Opcode = ARMISD::VMAXVu;
13102 if (CC == ISD::SETULT)
13103 std::swap(TrueVal, FalseVal);
13104 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMAX ||
13105 FalseVal->getOpcode() == ISD::VECREDUCE_SMAX) &&
13106 (CC == ISD::SETGT || CC == ISD::SETLT)) {
13107 Opcode = ARMISD::VMAXVs;
13108 if (CC == ISD::SETLT)
13109 std::swap(TrueVal, FalseVal);
13110 } else
13111 return SDValue();
13112
13113 // Normalise to the right hand side being the vector reduction
13114 switch (TrueVal->getOpcode()) {
13119 std::swap(LHS, RHS);
13120 std::swap(TrueVal, FalseVal);
13121 break;
13122 }
13123
13124 EVT VectorType = FalseVal->getOperand(0).getValueType();
13125
13126 if (VectorType != MVT::v16i8 && VectorType != MVT::v8i16 &&
13127 VectorType != MVT::v4i32)
13128 return SDValue();
13129
13130 EVT VectorScalarType = VectorType.getVectorElementType();
13131
13132 // The values being selected must also be the ones being compared
13133 if (TrueVal != LHS || FalseVal != RHS)
13134 return SDValue();
13135
13136 EVT LeftType = LHS->getValueType(0);
13137 EVT RightType = RHS->getValueType(0);
13138
13139 // The types must match the reduced type too
13140 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13141 return SDValue();
13142
13143 // Legalise the scalar to an i32
13144 if (VectorScalarType != MVT::i32)
13145 LHS = DCI.DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS);
13146
13147 // Generate the reduction as an i32 for legalisation purposes
13148 auto Reduction =
13149 DCI.DAG.getNode(Opcode, dl, MVT::i32, LHS, RHS->getOperand(0));
13150
13151 // The result isn't actually an i32 so truncate it back to its original type
13152 if (VectorScalarType != MVT::i32)
13153 Reduction = DCI.DAG.getNode(ISD::TRUNCATE, dl, VectorScalarType, Reduction);
13154
13155 return Reduction;
13156}
13157
13158// A special combine for the vqdmulh family of instructions. This is one of the
13159// potential set of patterns that could patch this instruction. The base pattern
13160// you would expect to be min(max(ashr(mul(mul(sext(x), 2), sext(y)), 16))).
13161// This matches the different min(max(ashr(mul(mul(sext(x), sext(y)), 2), 16))),
13162// which llvm will have optimized to min(ashr(mul(sext(x), sext(y)), 15))) as
13163// the max is unnecessary.
13165 EVT VT = N->getValueType(0);
13166 SDValue Shft;
13167 ConstantSDNode *Clamp;
13168
13169 if (!VT.isVector() || VT.getScalarSizeInBits() > 64)
13170 return SDValue();
13171
13172 if (N->getOpcode() == ISD::SMIN) {
13173 Shft = N->getOperand(0);
13174 Clamp = isConstOrConstSplat(N->getOperand(1));
13175 } else if (N->getOpcode() == ISD::VSELECT) {
13176 // Detect a SMIN, which for an i64 node will be a vselect/setcc, not a smin.
13177 SDValue Cmp = N->getOperand(0);
13178 if (Cmp.getOpcode() != ISD::SETCC ||
13179 Cmp.getOperandCondCode(2) != ISD::SETLT ||
13180 Cmp.getOperand(0) != N->getOperand(1) ||
13181 Cmp.getOperand(1) != N->getOperand(2))
13182 return SDValue();
13183 Shft = N->getOperand(1);
13184 Clamp = isConstOrConstSplat(N->getOperand(2));
13185 } else
13186 return SDValue();
13187
13188 if (!Clamp)
13189 return SDValue();
13190
13191 MVT ScalarType;
13192 int ShftAmt = 0;
13193 switch (Clamp->getSExtValue()) {
13194 case (1 << 7) - 1:
13195 ScalarType = MVT::i8;
13196 ShftAmt = 7;
13197 break;
13198 case (1 << 15) - 1:
13199 ScalarType = MVT::i16;
13200 ShftAmt = 15;
13201 break;
13202 case (1ULL << 31) - 1:
13203 ScalarType = MVT::i32;
13204 ShftAmt = 31;
13205 break;
13206 default:
13207 return SDValue();
13208 }
13209
13210 if (Shft.getOpcode() != ISD::SRA)
13211 return SDValue();
13213 if (!N1 || N1->getSExtValue() != ShftAmt)
13214 return SDValue();
13215
13216 SDValue Mul = Shft.getOperand(0);
13217 if (Mul.getOpcode() != ISD::MUL)
13218 return SDValue();
13219
13220 SDValue Ext0 = Mul.getOperand(0);
13221 SDValue Ext1 = Mul.getOperand(1);
13222 if (Ext0.getOpcode() != ISD::SIGN_EXTEND ||
13223 Ext1.getOpcode() != ISD::SIGN_EXTEND)
13224 return SDValue();
13225 EVT VecVT = Ext0.getOperand(0).getValueType();
13226 if (!VecVT.isPow2VectorType() || VecVT.getVectorNumElements() == 1)
13227 return SDValue();
13228 if (Ext1.getOperand(0).getValueType() != VecVT ||
13229 VecVT.getScalarType() != ScalarType ||
13230 VT.getScalarSizeInBits() < ScalarType.getScalarSizeInBits() * 2)
13231 return SDValue();
13232
13233 SDLoc DL(Mul);
13234 unsigned LegalLanes = 128 / (ShftAmt + 1);
13235 EVT LegalVecVT = MVT::getVectorVT(ScalarType, LegalLanes);
13236 // For types smaller than legal vectors extend to be legal and only use needed
13237 // lanes.
13238 if (VecVT.getSizeInBits() < 128) {
13239 EVT ExtVecVT =
13241 VecVT.getVectorNumElements());
13242 SDValue Inp0 =
13243 DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext0.getOperand(0));
13244 SDValue Inp1 =
13245 DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext1.getOperand(0));
13246 Inp0 = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, LegalVecVT, Inp0);
13247 Inp1 = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, LegalVecVT, Inp1);
13248 SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, LegalVecVT, Inp0, Inp1);
13249 SDValue Trunc = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, ExtVecVT, VQDMULH);
13250 Trunc = DAG.getNode(ISD::TRUNCATE, DL, VecVT, Trunc);
13251 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Trunc);
13252 }
13253
13254 // For larger types, split into legal sized chunks.
13255 assert(VecVT.getSizeInBits() % 128 == 0 && "Expected a power2 type");
13256 unsigned NumParts = VecVT.getSizeInBits() / 128;
13258 for (unsigned I = 0; I < NumParts; ++I) {
13259 SDValue Inp0 =
13260 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext0.getOperand(0),
13261 DAG.getVectorIdxConstant(I * LegalLanes, DL));
13262 SDValue Inp1 =
13263 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext1.getOperand(0),
13264 DAG.getVectorIdxConstant(I * LegalLanes, DL));
13265 SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, LegalVecVT, Inp0, Inp1);
13266 Parts.push_back(VQDMULH);
13267 }
13268 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT,
13269 DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Parts));
13270}
13271
13274 const ARMSubtarget *Subtarget) {
13275 if (!Subtarget->hasMVEIntegerOps())
13276 return SDValue();
13277
13278 // Constant fold vselect 0, A, B -> B
13279 // and vselect 0xffff, A, B -> A
13280 if (N->getOperand(0).getOpcode() == ARMISD::PREDICATE_CAST &&
13281 isa<ConstantSDNode>(N->getOperand(0).getOperand(0))) {
13282 unsigned C = N->getOperand(0).getConstantOperandVal(0);
13283 if (C == 0)
13284 return N->getOperand(2);
13285 if (C == 0xffff)
13286 return N->getOperand(1);
13287 }
13288
13289 if (SDValue V = PerformVQDMULHCombine(N, DCI.DAG))
13290 return V;
13291
13292 // Transforms vselect(not(cond), lhs, rhs) into vselect(cond, rhs, lhs).
13293 //
13294 // We need to re-implement this optimization here as the implementation in the
13295 // Target-Independent DAGCombiner does not handle the kind of constant we make
13296 // (it calls isConstOrConstSplat with AllowTruncation set to false - and for
13297 // good reason, allowing truncation there would break other targets).
13298 //
13299 // Currently, this is only done for MVE, as it's the only target that benefits
13300 // from this transformation (e.g. VPNOT+VPSEL becomes a single VPSEL).
13301 if (N->getOperand(0).getOpcode() != ISD::XOR)
13302 return SDValue();
13303 SDValue XOR = N->getOperand(0);
13304
13305 // Check if the XOR's RHS is either a 1, or a BUILD_VECTOR of 1s.
13306 // It is important to check with truncation allowed as the BUILD_VECTORs we
13307 // generate in those situations will truncate their operands.
13308 ConstantSDNode *Const =
13309 isConstOrConstSplat(XOR->getOperand(1), /*AllowUndefs*/ false,
13310 /*AllowTruncation*/ true);
13311 if (!Const || !Const->isOne())
13312 return SDValue();
13313
13314 // Rewrite into vselect(cond, rhs, lhs).
13315 SDValue Cond = XOR->getOperand(0);
13316 SDValue LHS = N->getOperand(1);
13317 SDValue RHS = N->getOperand(2);
13318 EVT Type = N->getValueType(0);
13319 return DCI.DAG.getNode(ISD::VSELECT, SDLoc(N), Type, Cond, RHS, LHS);
13320}
13321
13322// Convert vsetcc([0,1,2,..], splat(n), ult) -> vctp n
13325 const ARMSubtarget *Subtarget) {
13326 SDValue Op0 = N->getOperand(0);
13327 SDValue Op1 = N->getOperand(1);
13328 ISD::CondCode CC = N->getOperandCondCode(2);
13329 EVT VT = N->getValueType(0);
13330
13331 if (!Subtarget->hasMVEIntegerOps() ||
13333 return SDValue();
13334
13335 if (CC == ISD::SETUGT) {
13336 std::swap(Op0, Op1);
13337 CC = ISD::SETULT;
13338 }
13339
13340 if (CC != ISD::SETULT || VT.getScalarSizeInBits() != 1 ||
13342 return SDValue();
13343
13344 // Check first operand is BuildVector of 0,1,2,...
13345 for (unsigned I = 0; I < VT.getVectorNumElements(); I++) {
13346 if (!Op0.getOperand(I).isUndef() &&
13348 Op0.getConstantOperandVal(I) == I))
13349 return SDValue();
13350 }
13351
13352 // The second is a Splat of Op1S
13353 SDValue Op1S = DCI.DAG.getSplatValue(Op1);
13354 if (!Op1S)
13355 return SDValue();
13356
13357 unsigned Opc;
13358 switch (VT.getVectorNumElements()) {
13359 case 4:
13360 Opc = Intrinsic::arm_mve_vctp32;
13361 break;
13362 case 8:
13363 Opc = Intrinsic::arm_mve_vctp16;
13364 break;
13365 case 16:
13366 Opc = Intrinsic::arm_mve_vctp8;
13367 break;
13368 default:
13369 return SDValue();
13370 }
13371
13372 SDLoc DL(N);
13373 return DCI.DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
13374 DCI.DAG.getConstant(Opc, DL, MVT::i32),
13375 DCI.DAG.getZExtOrTrunc(Op1S, DL, MVT::i32));
13376}
13377
13378/// PerformADDECombine - Target-specific dag combine transform from
13379/// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or
13380/// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL
13383 const ARMSubtarget *Subtarget) {
13384 // Only ARM and Thumb2 support UMLAL/SMLAL.
13385 if (Subtarget->isThumb1Only())
13386 return PerformAddeSubeCombine(N, DCI, Subtarget);
13387
13388 // Only perform the checks after legalize when the pattern is available.
13389 if (DCI.isBeforeLegalize()) return SDValue();
13390
13391 return AddCombineTo64bitUMAAL(N, DCI, Subtarget);
13392}
13393
13394/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
13395/// operands N0 and N1. This is a helper for PerformADDCombine that is
13396/// called with the default operands, and if that fails, with commuted
13397/// operands.
13400 const ARMSubtarget *Subtarget){
13401 // Attempt to create vpadd for this add.
13402 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget))
13403 return Result;
13404
13405 // Attempt to create vpaddl for this add.
13406 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget))
13407 return Result;
13408 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI,
13409 Subtarget))
13410 return Result;
13411
13412 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
13413 if (N0.getNode()->hasOneUse())
13414 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI))
13415 return Result;
13416 return SDValue();
13417}
13418
13420 EVT VT = N->getValueType(0);
13421 SDValue N0 = N->getOperand(0);
13422 SDValue N1 = N->getOperand(1);
13423 SDLoc dl(N);
13424
13425 auto IsVecReduce = [](SDValue Op) {
13426 switch (Op.getOpcode()) {
13427 case ISD::VECREDUCE_ADD:
13428 case ARMISD::VADDVs:
13429 case ARMISD::VADDVu:
13430 case ARMISD::VMLAVs:
13431 case ARMISD::VMLAVu:
13432 return true;
13433 }
13434 return false;
13435 };
13436
13437 auto DistrubuteAddAddVecReduce = [&](SDValue N0, SDValue N1) {
13438 // Distribute add(X, add(vecreduce(Y), vecreduce(Z))) ->
13439 // add(add(X, vecreduce(Y)), vecreduce(Z))
13440 // to make better use of vaddva style instructions.
13441 if (VT == MVT::i32 && N1.getOpcode() == ISD::ADD && !IsVecReduce(N0) &&
13442 IsVecReduce(N1.getOperand(0)) && IsVecReduce(N1.getOperand(1)) &&
13443 !isa<ConstantSDNode>(N0) && N1->hasOneUse()) {
13444 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0, N1.getOperand(0));
13445 return DAG.getNode(ISD::ADD, dl, VT, Add0, N1.getOperand(1));
13446 }
13447 // And turn add(add(A, reduce(B)), add(C, reduce(D))) ->
13448 // add(add(add(A, C), reduce(B)), reduce(D))
13449 if (VT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
13450 N1.getOpcode() == ISD::ADD && N0->hasOneUse() && N1->hasOneUse()) {
13451 unsigned N0RedOp = 0;
13452 if (!IsVecReduce(N0.getOperand(N0RedOp))) {
13453 N0RedOp = 1;
13454 if (!IsVecReduce(N0.getOperand(N0RedOp)))
13455 return SDValue();
13456 }
13457
13458 unsigned N1RedOp = 0;
13459 if (!IsVecReduce(N1.getOperand(N1RedOp)))
13460 N1RedOp = 1;
13461 if (!IsVecReduce(N1.getOperand(N1RedOp)))
13462 return SDValue();
13463
13464 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0.getOperand(1 - N0RedOp),
13465 N1.getOperand(1 - N1RedOp));
13466 SDValue Add1 =
13467 DAG.getNode(ISD::ADD, dl, VT, Add0, N0.getOperand(N0RedOp));
13468 return DAG.getNode(ISD::ADD, dl, VT, Add1, N1.getOperand(N1RedOp));
13469 }
13470 return SDValue();
13471 };
13472 if (SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13473 return R;
13474 if (SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13475 return R;
13476
13477 // Distribute add(vecreduce(load(Y)), vecreduce(load(Z)))
13478 // Or add(add(X, vecreduce(load(Y))), vecreduce(load(Z)))
13479 // by ascending load offsets. This can help cores prefetch if the order of
13480 // loads is more predictable.
13481 auto DistrubuteVecReduceLoad = [&](SDValue N0, SDValue N1, bool IsForward) {
13482 // Check if two reductions are known to load data where one is before/after
13483 // another. Return negative if N0 loads data before N1, positive if N1 is
13484 // before N0 and 0 otherwise if nothing is known.
13485 auto IsKnownOrderedLoad = [&](SDValue N0, SDValue N1) {
13486 // Look through to the first operand of a MUL, for the VMLA case.
13487 // Currently only looks at the first operand, in the hope they are equal.
13488 if (N0.getOpcode() == ISD::MUL)
13489 N0 = N0.getOperand(0);
13490 if (N1.getOpcode() == ISD::MUL)
13491 N1 = N1.getOperand(0);
13492
13493 // Return true if the two operands are loads to the same object and the
13494 // offset of the first is known to be less than the offset of the second.
13495 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
13496 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
13497 if (!Load0 || !Load1 || Load0->getChain() != Load1->getChain() ||
13498 !Load0->isSimple() || !Load1->isSimple() || Load0->isIndexed() ||
13499 Load1->isIndexed())
13500 return 0;
13501
13502 auto BaseLocDecomp0 = BaseIndexOffset::match(Load0, DAG);
13503 auto BaseLocDecomp1 = BaseIndexOffset::match(Load1, DAG);
13504
13505 if (!BaseLocDecomp0.getBase() ||
13506 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13507 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13508 return 0;
13509 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13510 return -1;
13511 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13512 return 1;
13513 return 0;
13514 };
13515
13516 SDValue X;
13517 if (N0.getOpcode() == ISD::ADD && N0->hasOneUse()) {
13518 if (IsVecReduce(N0.getOperand(0)) && IsVecReduce(N0.getOperand(1))) {
13519 int IsBefore = IsKnownOrderedLoad(N0.getOperand(0).getOperand(0),
13520 N0.getOperand(1).getOperand(0));
13521 if (IsBefore < 0) {
13522 X = N0.getOperand(0);
13523 N0 = N0.getOperand(1);
13524 } else if (IsBefore > 0) {
13525 X = N0.getOperand(1);
13526 N0 = N0.getOperand(0);
13527 } else
13528 return SDValue();
13529 } else if (IsVecReduce(N0.getOperand(0))) {
13530 X = N0.getOperand(1);
13531 N0 = N0.getOperand(0);
13532 } else if (IsVecReduce(N0.getOperand(1))) {
13533 X = N0.getOperand(0);
13534 N0 = N0.getOperand(1);
13535 } else
13536 return SDValue();
13537 } else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13538 IsKnownOrderedLoad(N0.getOperand(0), N1.getOperand(0)) < 0) {
13539 // Note this is backward to how you would expect. We create
13540 // add(reduce(load + 16), reduce(load + 0)) so that the
13541 // add(reduce(load+16), X) is combined into VADDVA(X, load+16)), leaving
13542 // the X as VADDV(load + 0)
13543 return DAG.getNode(ISD::ADD, dl, VT, N1, N0);
13544 } else
13545 return SDValue();
13546
13547 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13548 return SDValue();
13549
13550 if (IsKnownOrderedLoad(N1.getOperand(0), N0.getOperand(0)) >= 0)
13551 return SDValue();
13552
13553 // Switch from add(add(X, N0), N1) to add(add(X, N1), N0)
13554 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, X, N1);
13555 return DAG.getNode(ISD::ADD, dl, VT, Add0, N0);
13556 };
13557 if (SDValue R = DistrubuteVecReduceLoad(N0, N1, true))
13558 return R;
13559 if (SDValue R = DistrubuteVecReduceLoad(N1, N0, false))
13560 return R;
13561 return SDValue();
13562}
13563
13565 const ARMSubtarget *Subtarget) {
13566 if (!Subtarget->hasMVEIntegerOps())
13567 return SDValue();
13568
13570 return R;
13571
13572 EVT VT = N->getValueType(0);
13573 SDValue N0 = N->getOperand(0);
13574 SDValue N1 = N->getOperand(1);
13575 SDLoc dl(N);
13576
13577 if (VT != MVT::i64)
13578 return SDValue();
13579
13580 // We are looking for a i64 add of a VADDLVx. Due to these being i64's, this
13581 // will look like:
13582 // t1: i32,i32 = ARMISD::VADDLVs x
13583 // t2: i64 = build_pair t1, t1:1
13584 // t3: i64 = add t2, y
13585 // Otherwise we try to push the add up above VADDLVAx, to potentially allow
13586 // the add to be simplified separately.
13587 // We also need to check for sext / zext and commutitive adds.
13588 auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA,
13589 SDValue NB) {
13590 if (NB->getOpcode() != ISD::BUILD_PAIR)
13591 return SDValue();
13592 SDValue VecRed = NB->getOperand(0);
13593 if ((VecRed->getOpcode() != Opcode && VecRed->getOpcode() != OpcodeA) ||
13594 VecRed.getResNo() != 0 ||
13595 NB->getOperand(1) != SDValue(VecRed.getNode(), 1))
13596 return SDValue();
13597
13598 if (VecRed->getOpcode() == OpcodeA) {
13599 // add(NA, VADDLVA(Inp), Y) -> VADDLVA(add(NA, Inp), Y)
13600 SDValue Inp = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64,
13601 VecRed.getOperand(0), VecRed.getOperand(1));
13602 NA = DAG.getNode(ISD::ADD, dl, MVT::i64, Inp, NA);
13603 }
13604
13606 std::tie(Ops[0], Ops[1]) = DAG.SplitScalar(NA, dl, MVT::i32, MVT::i32);
13607
13608 unsigned S = VecRed->getOpcode() == OpcodeA ? 2 : 0;
13609 for (unsigned I = S, E = VecRed.getNumOperands(); I < E; I++)
13610 Ops.push_back(VecRed->getOperand(I));
13611 SDValue Red =
13612 DAG.getNode(OpcodeA, dl, DAG.getVTList({MVT::i32, MVT::i32}), Ops);
13613 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Red,
13614 SDValue(Red.getNode(), 1));
13615 };
13616
13617 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1))
13618 return M;
13619 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N0, N1))
13620 return M;
13621 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N1, N0))
13622 return M;
13623 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N1, N0))
13624 return M;
13625 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N0, N1))
13626 return M;
13627 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N0, N1))
13628 return M;
13629 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N1, N0))
13630 return M;
13631 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N1, N0))
13632 return M;
13633 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N0, N1))
13634 return M;
13635 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N0, N1))
13636 return M;
13637 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N1, N0))
13638 return M;
13639 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
13640 return M;
13641 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N0, N1))
13642 return M;
13643 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N0, N1))
13644 return M;
13645 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N1, N0))
13646 return M;
13647 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N1, N0))
13648 return M;
13649 return SDValue();
13650}
13651
13652bool
13654 CombineLevel Level) const {
13655 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
13656 N->getOpcode() == ISD::SRL) &&
13657 "Expected shift op");
13658
13659 SDValue ShiftLHS = N->getOperand(0);
13660 if (!ShiftLHS->hasOneUse())
13661 return false;
13662
13663 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
13664 !ShiftLHS.getOperand(0)->hasOneUse())
13665 return false;
13666
13667 if (Level == BeforeLegalizeTypes)
13668 return true;
13669
13670 if (N->getOpcode() != ISD::SHL)
13671 return true;
13672
13673 if (Subtarget->isThumb1Only()) {
13674 // Avoid making expensive immediates by commuting shifts. (This logic
13675 // only applies to Thumb1 because ARM and Thumb2 immediates can be shifted
13676 // for free.)
13677 if (N->getOpcode() != ISD::SHL)
13678 return true;
13679 SDValue N1 = N->getOperand(0);
13680 if (N1->getOpcode() != ISD::ADD && N1->getOpcode() != ISD::AND &&
13681 N1->getOpcode() != ISD::OR && N1->getOpcode() != ISD::XOR)
13682 return true;
13683 if (auto *Const = dyn_cast<ConstantSDNode>(N1->getOperand(1))) {
13684 if (Const->getAPIntValue().ult(256))
13685 return false;
13686 if (N1->getOpcode() == ISD::ADD && Const->getAPIntValue().slt(0) &&
13687 Const->getAPIntValue().sgt(-256))
13688 return false;
13689 }
13690 return true;
13691 }
13692
13693 // Turn off commute-with-shift transform after legalization, so it doesn't
13694 // conflict with PerformSHLSimplify. (We could try to detect when
13695 // PerformSHLSimplify would trigger more precisely, but it isn't
13696 // really necessary.)
13697 return false;
13698}
13699
13701 const SDNode *N) const {
13702 assert(N->getOpcode() == ISD::XOR &&
13703 (N->getOperand(0).getOpcode() == ISD::SHL ||
13704 N->getOperand(0).getOpcode() == ISD::SRL) &&
13705 "Expected XOR(SHIFT) pattern");
13706
13707 // Only commute if the entire NOT mask is a hidden shifted mask.
13708 auto *XorC = dyn_cast<ConstantSDNode>(N->getOperand(1));
13709 auto *ShiftC = dyn_cast<ConstantSDNode>(N->getOperand(0).getOperand(1));
13710 if (XorC && ShiftC) {
13711 unsigned MaskIdx, MaskLen;
13712 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13713 unsigned ShiftAmt = ShiftC->getZExtValue();
13714 unsigned BitWidth = N->getValueType(0).getScalarSizeInBits();
13715 if (N->getOperand(0).getOpcode() == ISD::SHL)
13716 return MaskIdx == ShiftAmt && MaskLen == (BitWidth - ShiftAmt);
13717 return MaskIdx == 0 && MaskLen == (BitWidth - ShiftAmt);
13718 }
13719 }
13720
13721 return false;
13722}
13723
13725 const SDNode *N) const {
13726 assert(((N->getOpcode() == ISD::SHL &&
13727 N->getOperand(0).getOpcode() == ISD::SRL) ||
13728 (N->getOpcode() == ISD::SRL &&
13729 N->getOperand(0).getOpcode() == ISD::SHL)) &&
13730 "Expected shift-shift mask");
13731
13732 if (!Subtarget->isThumb1Only())
13733 return true;
13734
13735 EVT VT = N->getValueType(0);
13736 if (VT.getScalarSizeInBits() > 32)
13737 return true;
13738
13739 return false;
13740}
13741
13743 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
13744 SDValue Y) const {
13745 return Subtarget->hasMVEIntegerOps() && isTypeLegal(VT) &&
13746 SelectOpcode == ISD::VSELECT;
13747}
13748
13750 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps()) {
13751 if (Subtarget->isThumb1Only())
13752 return VT.getScalarSizeInBits() <= 32;
13753 return true;
13754 }
13755 return VT.isScalarInteger();
13756}
13757
13759 EVT VT) const {
13760 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
13761 return false;
13762
13763 switch (FPVT.getSimpleVT().SimpleTy) {
13764 case MVT::f16:
13765 return Subtarget->hasVFP2Base();
13766 case MVT::f32:
13767 return Subtarget->hasVFP2Base();
13768 case MVT::f64:
13769 return Subtarget->hasFP64();
13770 case MVT::v4f32:
13771 case MVT::v8f16:
13772 return Subtarget->hasMVEFloatOps();
13773 default:
13774 return false;
13775 }
13776}
13777
13780 const ARMSubtarget *ST) {
13781 // Allow the generic combiner to identify potential bswaps.
13782 if (DCI.isBeforeLegalize())
13783 return SDValue();
13784
13785 // DAG combiner will fold:
13786 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
13787 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2
13788 // Other code patterns that can be also be modified have the following form:
13789 // b + ((a << 1) | 510)
13790 // b + ((a << 1) & 510)
13791 // b + ((a << 1) ^ 510)
13792 // b + ((a << 1) + 510)
13793
13794 // Many instructions can perform the shift for free, but it requires both
13795 // the operands to be registers. If c1 << c2 is too large, a mov immediate
13796 // instruction will needed. So, unfold back to the original pattern if:
13797 // - if c1 and c2 are small enough that they don't require mov imms.
13798 // - the user(s) of the node can perform an shl
13799
13800 // No shifted operands for 16-bit instructions.
13801 if (ST->isThumb1Only())
13802 return SDValue();
13803
13804 // Check that all the users could perform the shl themselves.
13805 for (auto *U : N->users()) {
13806 switch(U->getOpcode()) {
13807 default:
13808 return SDValue();
13809 case ISD::SUB:
13810 case ISD::ADD:
13811 case ISD::AND:
13812 case ISD::OR:
13813 case ISD::XOR:
13814 case ISD::SETCC:
13815 case ARMISD::CMP:
13816 // Check that the user isn't already using a constant because there
13817 // aren't any instructions that support an immediate operand and a
13818 // shifted operand.
13819 if (isa<ConstantSDNode>(U->getOperand(0)) ||
13820 isa<ConstantSDNode>(U->getOperand(1)))
13821 return SDValue();
13822
13823 // Check that it's not already using a shift.
13824 if (U->getOperand(0).getOpcode() == ISD::SHL ||
13825 U->getOperand(1).getOpcode() == ISD::SHL)
13826 return SDValue();
13827 break;
13828 }
13829 }
13830
13831 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR &&
13832 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND)
13833 return SDValue();
13834
13835 if (N->getOperand(0).getOpcode() != ISD::SHL)
13836 return SDValue();
13837
13838 SDValue SHL = N->getOperand(0);
13839
13840 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
13841 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
13842 if (!C1ShlC2 || !C2)
13843 return SDValue();
13844
13845 APInt C2Int = C2->getAPIntValue();
13846 APInt C1Int = C1ShlC2->getAPIntValue();
13847 unsigned C2Width = C2Int.getBitWidth();
13848 if (C2Int.uge(C2Width))
13849 return SDValue();
13850 uint64_t C2Value = C2Int.getZExtValue();
13851
13852 // Check that performing a lshr will not lose any information.
13853 APInt Mask = APInt::getHighBitsSet(C2Width, C2Width - C2Value);
13854 if ((C1Int & Mask) != C1Int)
13855 return SDValue();
13856
13857 // Shift the first constant.
13858 C1Int.lshrInPlace(C2Int);
13859
13860 // The immediates are encoded as an 8-bit value that can be rotated.
13861 auto LargeImm = [](const APInt &Imm) {
13862 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
13863 return Imm.getBitWidth() - Zeros > 8;
13864 };
13865
13866 if (LargeImm(C1Int) || LargeImm(C2Int))
13867 return SDValue();
13868
13869 SelectionDAG &DAG = DCI.DAG;
13870 SDLoc dl(N);
13871 SDValue X = SHL.getOperand(0);
13872 SDValue BinOp = DAG.getNode(N->getOpcode(), dl, MVT::i32, X,
13873 DAG.getConstant(C1Int, dl, MVT::i32));
13874 // Shift left to compensate for the lshr of C1Int.
13875 SDValue Res = DAG.getNode(ISD::SHL, dl, MVT::i32, BinOp, SHL.getOperand(1));
13876
13877 LLVM_DEBUG(dbgs() << "Simplify shl use:\n"; SHL.getOperand(0).dump();
13878 SHL.dump(); N->dump());
13879 LLVM_DEBUG(dbgs() << "Into:\n"; X.dump(); BinOp.dump(); Res.dump());
13880 return Res;
13881}
13882
13883
13884/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
13885///
13888 const ARMSubtarget *Subtarget) {
13889 SDValue N0 = N->getOperand(0);
13890 SDValue N1 = N->getOperand(1);
13891
13892 // Only works one way, because it needs an immediate operand.
13893 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
13894 return Result;
13895
13896 if (SDValue Result = PerformADDVecReduce(N, DCI.DAG, Subtarget))
13897 return Result;
13898
13899 // First try with the default operand order.
13900 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget))
13901 return Result;
13902
13903 // If that didn't work, try again with the operands commuted.
13904 return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget);
13905}
13906
13907// Combine (sub 0, (csinc X, Y, CC)) -> (csinv -X, Y, CC)
13908// providing -X is as cheap as X (currently, just a constant).
13910 if (N->getValueType(0) != MVT::i32 || !isNullConstant(N->getOperand(0)))
13911 return SDValue();
13912 SDValue CSINC = N->getOperand(1);
13913 if (CSINC.getOpcode() != ARMISD::CSINC || !CSINC.hasOneUse())
13914 return SDValue();
13915
13917 if (!X)
13918 return SDValue();
13919
13920 return DAG.getNode(ARMISD::CSINV, SDLoc(N), MVT::i32,
13921 DAG.getNode(ISD::SUB, SDLoc(N), MVT::i32, N->getOperand(0),
13922 CSINC.getOperand(0)),
13923 CSINC.getOperand(1), CSINC.getOperand(2),
13924 CSINC.getOperand(3));
13925}
13926
13928 // Free to negate.
13930 return 0;
13931
13932 // Will save one instruction.
13933 if (Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)))
13934 return -1;
13935
13936 // Can freely negate by converting sra <-> srl.
13937 if (Op.getOpcode() == ISD::SRA || Op.getOpcode() == ISD::SRL) {
13938 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Op.getOperand(1));
13939 if (Op.hasOneUse() && ShiftAmt &&
13940 ShiftAmt->getZExtValue() == Op.getValueType().getScalarSizeInBits() - 1)
13941 return 0;
13942 }
13943
13944 // Will have to create sub.
13945 return 1;
13946}
13947
13948// Try to fold
13949//
13950// (neg (cmov X, Y)) -> (cmov (neg X), (neg Y))
13951//
13952// The folding helps cmov to be matched with csneg without generating
13953// redundant neg instruction.
13955 assert(N->getOpcode() == ISD::SUB);
13956 if (!isNullConstant(N->getOperand(0)))
13957 return SDValue();
13958
13959 SDValue CMov = N->getOperand(1);
13960 if (CMov.getOpcode() != ARMISD::CMOV || !CMov->hasOneUse())
13961 return SDValue();
13962
13963 SDValue N0 = CMov.getOperand(0);
13964 SDValue N1 = CMov.getOperand(1);
13965
13966 // Only perform the fold if we actually save something.
13967 if (getNegationCost(N0) + getNegationCost(N1) > 0)
13968 return SDValue();
13969
13970 SDLoc DL(N);
13971 EVT VT = CMov.getValueType();
13972
13973 SDValue N0N = DAG.getNegative(N0, DL, VT);
13974 SDValue N1N = DAG.getNegative(N1, DL, VT);
13975 return DAG.getNode(ARMISD::CMOV, DL, VT, N0N, N1N, CMov.getOperand(2),
13976 CMov.getOperand(3));
13977}
13978
13979/// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
13980///
13983 const ARMSubtarget *Subtarget) {
13984 SDValue N0 = N->getOperand(0);
13985 SDValue N1 = N->getOperand(1);
13986
13987 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
13988 if (N1.getNode()->hasOneUse())
13989 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI))
13990 return Result;
13991
13992 if (SDValue R = PerformSubCSINCCombine(N, DCI.DAG))
13993 return R;
13994
13995 if (SDValue Val = performNegCMovCombine(N, DCI.DAG))
13996 return Val;
13997
13998 if (!Subtarget->hasMVEIntegerOps() || !N->getValueType(0).isVector())
13999 return SDValue();
14000
14001 // Fold (sub (ARMvmovImm 0), (ARMvdup x)) -> (ARMvdup (sub 0, x))
14002 // so that we can readily pattern match more mve instructions which can use
14003 // a scalar operand.
14004 SDValue VDup = N->getOperand(1);
14005 if (VDup->getOpcode() != ARMISD::VDUP)
14006 return SDValue();
14007
14008 SDValue VMov = N->getOperand(0);
14009 if (VMov->getOpcode() == ISD::BITCAST)
14010 VMov = VMov->getOperand(0);
14011
14012 if (VMov->getOpcode() != ARMISD::VMOVIMM || !isZeroVector(VMov))
14013 return SDValue();
14014
14015 SDLoc dl(N);
14016 SDValue Negate = DCI.DAG.getNode(ISD::SUB, dl, MVT::i32,
14017 DCI.DAG.getConstant(0, dl, MVT::i32),
14018 VDup->getOperand(0));
14019 return DCI.DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), Negate);
14020}
14021
14022/// PerformVMULCombine
14023/// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the
14024/// special multiplier accumulator forwarding.
14025/// vmul d3, d0, d2
14026/// vmla d3, d1, d2
14027/// is faster than
14028/// vadd d3, d0, d1
14029/// vmul d3, d3, d2
14030// However, for (A + B) * (A + B),
14031// vadd d2, d0, d1
14032// vmul d3, d0, d2
14033// vmla d3, d1, d2
14034// is slower than
14035// vadd d2, d0, d1
14036// vmul d3, d2, d2
14039 const ARMSubtarget *Subtarget) {
14040 if (!Subtarget->hasVMLxForwarding())
14041 return SDValue();
14042
14043 SelectionDAG &DAG = DCI.DAG;
14044 SDValue N0 = N->getOperand(0);
14045 SDValue N1 = N->getOperand(1);
14046 unsigned Opcode = N0.getOpcode();
14047 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
14048 Opcode != ISD::FADD && Opcode != ISD::FSUB) {
14049 Opcode = N1.getOpcode();
14050 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
14051 Opcode != ISD::FADD && Opcode != ISD::FSUB)
14052 return SDValue();
14053 std::swap(N0, N1);
14054 }
14055
14056 if (N0 == N1)
14057 return SDValue();
14058
14059 EVT VT = N->getValueType(0);
14060 SDLoc DL(N);
14061 SDValue N00 = N0->getOperand(0);
14062 SDValue N01 = N0->getOperand(1);
14063 return DAG.getNode(Opcode, DL, VT,
14064 DAG.getNode(ISD::MUL, DL, VT, N00, N1),
14065 DAG.getNode(ISD::MUL, DL, VT, N01, N1));
14066}
14067
14069 const ARMSubtarget *Subtarget) {
14070 EVT VT = N->getValueType(0);
14071 if (VT != MVT::v2i64)
14072 return SDValue();
14073
14074 SDValue N0 = N->getOperand(0);
14075 SDValue N1 = N->getOperand(1);
14076
14077 auto IsSignExt = [&](SDValue Op) {
14078 if (Op->getOpcode() != ISD::SIGN_EXTEND_INREG)
14079 return SDValue();
14080 EVT VT = cast<VTSDNode>(Op->getOperand(1))->getVT();
14081 if (VT.getScalarSizeInBits() == 32)
14082 return Op->getOperand(0);
14083 return SDValue();
14084 };
14085 auto IsZeroExt = [&](SDValue Op) {
14086 // Zero extends are a little more awkward. At the point we are matching
14087 // this, we are looking for an AND with a (-1, 0, -1, 0) buildvector mask.
14088 // That might be before of after a bitcast depending on how the and is
14089 // placed. Because this has to look through bitcasts, it is currently only
14090 // supported on LE.
14091 if (!Subtarget->isLittle())
14092 return SDValue();
14093
14094 SDValue And = Op;
14095 if (And->getOpcode() == ISD::BITCAST)
14096 And = And->getOperand(0);
14097 if (And->getOpcode() != ISD::AND)
14098 return SDValue();
14099 SDValue Mask = And->getOperand(1);
14100 if (Mask->getOpcode() == ISD::BITCAST)
14101 Mask = Mask->getOperand(0);
14102
14103 if (Mask->getOpcode() != ISD::BUILD_VECTOR ||
14104 Mask.getValueType() != MVT::v4i32)
14105 return SDValue();
14106 if (isAllOnesConstant(Mask->getOperand(0)) &&
14107 isNullConstant(Mask->getOperand(1)) &&
14108 isAllOnesConstant(Mask->getOperand(2)) &&
14109 isNullConstant(Mask->getOperand(3)))
14110 return And->getOperand(0);
14111 return SDValue();
14112 };
14113
14114 SDLoc dl(N);
14115 if (SDValue Op0 = IsSignExt(N0)) {
14116 if (SDValue Op1 = IsSignExt(N1)) {
14117 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
14118 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
14119 return DAG.getNode(ARMISD::VMULLs, dl, VT, New0a, New1a);
14120 }
14121 }
14122 if (SDValue Op0 = IsZeroExt(N0)) {
14123 if (SDValue Op1 = IsZeroExt(N1)) {
14124 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
14125 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
14126 return DAG.getNode(ARMISD::VMULLu, dl, VT, New0a, New1a);
14127 }
14128 }
14129
14130 return SDValue();
14131}
14132
14135 const ARMSubtarget *Subtarget) {
14136 SelectionDAG &DAG = DCI.DAG;
14137
14138 EVT VT = N->getValueType(0);
14139 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14140 return PerformMVEVMULLCombine(N, DAG, Subtarget);
14141
14142 if (Subtarget->isThumb1Only())
14143 return SDValue();
14144
14145 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14146 return SDValue();
14147
14148 if (VT.is64BitVector() || VT.is128BitVector())
14149 return PerformVMULCombine(N, DCI, Subtarget);
14150 if (VT != MVT::i32)
14151 return SDValue();
14152
14153 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
14154 if (!C)
14155 return SDValue();
14156
14157 int64_t MulAmt = C->getSExtValue();
14158 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(MulAmt);
14159
14160 ShiftAmt = ShiftAmt & (32 - 1);
14161 SDValue V = N->getOperand(0);
14162 SDLoc DL(N);
14163
14164 SDValue Res;
14165 MulAmt >>= ShiftAmt;
14166
14167 if (MulAmt >= 0) {
14168 if (llvm::has_single_bit<uint32_t>(MulAmt - 1)) {
14169 // (mul x, 2^N + 1) => (add (shl x, N), x)
14170 Res = DAG.getNode(ISD::ADD, DL, VT,
14171 V,
14172 DAG.getNode(ISD::SHL, DL, VT,
14173 V,
14174 DAG.getConstant(Log2_32(MulAmt - 1), DL,
14175 MVT::i32)));
14176 } else if (llvm::has_single_bit<uint32_t>(MulAmt + 1)) {
14177 // (mul x, 2^N - 1) => (sub (shl x, N), x)
14178 Res = DAG.getNode(ISD::SUB, DL, VT,
14179 DAG.getNode(ISD::SHL, DL, VT,
14180 V,
14181 DAG.getConstant(Log2_32(MulAmt + 1), DL,
14182 MVT::i32)),
14183 V);
14184 } else
14185 return SDValue();
14186 } else {
14187 uint64_t MulAmtAbs = -MulAmt;
14188 if (llvm::has_single_bit<uint32_t>(MulAmtAbs + 1)) {
14189 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
14190 Res = DAG.getNode(ISD::SUB, DL, VT,
14191 V,
14192 DAG.getNode(ISD::SHL, DL, VT,
14193 V,
14194 DAG.getConstant(Log2_32(MulAmtAbs + 1), DL,
14195 MVT::i32)));
14196 } else if (llvm::has_single_bit<uint32_t>(MulAmtAbs - 1)) {
14197 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
14198 Res = DAG.getNode(ISD::ADD, DL, VT,
14199 V,
14200 DAG.getNode(ISD::SHL, DL, VT,
14201 V,
14202 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL,
14203 MVT::i32)));
14204 Res = DAG.getNode(ISD::SUB, DL, VT,
14205 DAG.getConstant(0, DL, MVT::i32), Res);
14206 } else
14207 return SDValue();
14208 }
14209
14210 if (ShiftAmt != 0)
14211 Res = DAG.getNode(ISD::SHL, DL, VT,
14212 Res, DAG.getConstant(ShiftAmt, DL, MVT::i32));
14213
14214 // Do not add new nodes to DAG combiner worklist.
14215 DCI.CombineTo(N, Res, false);
14216 return SDValue();
14217}
14218
14221 const ARMSubtarget *Subtarget) {
14222 // Allow DAGCombine to pattern-match before we touch the canonical form.
14223 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14224 return SDValue();
14225
14226 if (N->getValueType(0) != MVT::i32)
14227 return SDValue();
14228
14229 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
14230 if (!N1C)
14231 return SDValue();
14232
14233 uint32_t C1 = (uint32_t)N1C->getZExtValue();
14234 // Don't transform uxtb/uxth.
14235 if (C1 == 255 || C1 == 65535)
14236 return SDValue();
14237
14238 SDNode *N0 = N->getOperand(0).getNode();
14239 if (!N0->hasOneUse())
14240 return SDValue();
14241
14242 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL)
14243 return SDValue();
14244
14245 bool LeftShift = N0->getOpcode() == ISD::SHL;
14246
14248 if (!N01C)
14249 return SDValue();
14250
14251 uint32_t C2 = (uint32_t)N01C->getZExtValue();
14252 if (!C2 || C2 >= 32)
14253 return SDValue();
14254
14255 // Clear irrelevant bits in the mask.
14256 if (LeftShift)
14257 C1 &= (-1U << C2);
14258 else
14259 C1 &= (-1U >> C2);
14260
14261 SelectionDAG &DAG = DCI.DAG;
14262 SDLoc DL(N);
14263
14264 // We have a pattern of the form "(and (shl x, c2) c1)" or
14265 // "(and (srl x, c2) c1)", where c1 is a shifted mask. Try to
14266 // transform to a pair of shifts, to save materializing c1.
14267
14268 // First pattern: right shift, then mask off leading bits.
14269 // FIXME: Use demanded bits?
14270 if (!LeftShift && isMask_32(C1)) {
14272 if (C2 < C3) {
14273 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
14274 DAG.getConstant(C3 - C2, DL, MVT::i32));
14275 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL,
14276 DAG.getConstant(C3, DL, MVT::i32));
14277 }
14278 }
14279
14280 // First pattern, reversed: left shift, then mask off trailing bits.
14281 if (LeftShift && isMask_32(~C1)) {
14283 if (C2 < C3) {
14284 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
14285 DAG.getConstant(C3 - C2, DL, MVT::i32));
14286 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
14287 DAG.getConstant(C3, DL, MVT::i32));
14288 }
14289 }
14290
14291 // Second pattern: left shift, then mask off leading bits.
14292 // FIXME: Use demanded bits?
14293 if (LeftShift && isShiftedMask_32(C1)) {
14294 uint32_t Trailing = llvm::countr_zero(C1);
14296 if (Trailing == C2 && C2 + C3 < 32) {
14297 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
14298 DAG.getConstant(C2 + C3, DL, MVT::i32));
14299 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL,
14300 DAG.getConstant(C3, DL, MVT::i32));
14301 }
14302 }
14303
14304 // Second pattern, reversed: right shift, then mask off trailing bits.
14305 // FIXME: Handle other patterns of known/demanded bits.
14306 if (!LeftShift && isShiftedMask_32(C1)) {
14307 uint32_t Leading = llvm::countl_zero(C1);
14309 if (Leading == C2 && C2 + C3 < 32) {
14310 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
14311 DAG.getConstant(C2 + C3, DL, MVT::i32));
14312 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
14313 DAG.getConstant(C3, DL, MVT::i32));
14314 }
14315 }
14316
14317 // Transform "(and (shl x, c2) c1)" into "(shl (and x, c1>>c2), c2)"
14318 // if "c1 >> c2" is a cheaper immediate than "c1"
14319 if (LeftShift &&
14321
14322 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i32, N0->getOperand(0),
14323 DAG.getConstant(C1 >> C2, DL, MVT::i32));
14324 return DAG.getNode(ISD::SHL, DL, MVT::i32, And,
14325 DAG.getConstant(C2, DL, MVT::i32));
14326 }
14327
14328 return SDValue();
14329}
14330
14333 const ARMSubtarget *Subtarget) {
14334 // Attempt to use immediate-form VBIC
14335 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
14336 SDLoc dl(N);
14337 EVT VT = N->getValueType(0);
14338 SelectionDAG &DAG = DCI.DAG;
14339
14340 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 ||
14341 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14342 return SDValue();
14343
14344 APInt SplatBits, SplatUndef;
14345 unsigned SplatBitSize;
14346 bool HasAnyUndefs;
14347 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14348 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14349 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14350 SplatBitSize == 64) {
14351 EVT VbicVT;
14352 SDValue Val = isVMOVModifiedImm((~SplatBits).getZExtValue(),
14353 SplatUndef.getZExtValue(), SplatBitSize,
14354 DAG, dl, VbicVT, VT, OtherModImm);
14355 if (Val.getNode()) {
14356 SDValue Input =
14357 DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VbicVT, N->getOperand(0));
14358 SDValue Vbic = DAG.getNode(ARMISD::VBICIMM, dl, VbicVT, Input, Val);
14359 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vbic);
14360 }
14361 }
14362 }
14363
14364 if (!Subtarget->isThumb1Only()) {
14365 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))
14366 if (SDValue Result = combineSelectAndUseCommutative(N, true, DCI))
14367 return Result;
14368
14369 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
14370 return Result;
14371 }
14372
14373 if (Subtarget->isThumb1Only())
14374 if (SDValue Result = CombineANDShift(N, DCI, Subtarget))
14375 return Result;
14376
14377 return SDValue();
14378}
14379
14380// Try combining OR nodes to SMULWB, SMULWT.
14383 const ARMSubtarget *Subtarget) {
14384 if (!Subtarget->hasV6Ops() ||
14385 (Subtarget->isThumb() &&
14386 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14387 return SDValue();
14388
14389 SDValue SRL = OR->getOperand(0);
14390 SDValue SHL = OR->getOperand(1);
14391
14392 if (SRL.getOpcode() != ISD::SRL || SHL.getOpcode() != ISD::SHL) {
14393 SRL = OR->getOperand(1);
14394 SHL = OR->getOperand(0);
14395 }
14396 if (!isSRL16(SRL) || !isSHL16(SHL))
14397 return SDValue();
14398
14399 // The first operands to the shifts need to be the two results from the
14400 // same smul_lohi node.
14401 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14402 SRL.getOperand(0).getOpcode() != ISD::SMUL_LOHI)
14403 return SDValue();
14404
14405 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14406 if (SRL.getOperand(0) != SDValue(SMULLOHI, 0) ||
14407 SHL.getOperand(0) != SDValue(SMULLOHI, 1))
14408 return SDValue();
14409
14410 // Now we have:
14411 // (or (srl (smul_lohi ?, ?), 16), (shl (smul_lohi ?, ?), 16)))
14412 // For SMUL[B|T] smul_lohi will take a 32-bit and a 16-bit arguments.
14413 // For SMUWB the 16-bit value will signed extended somehow.
14414 // For SMULWT only the SRA is required.
14415 // Check both sides of SMUL_LOHI
14416 SDValue OpS16 = SMULLOHI->getOperand(0);
14417 SDValue OpS32 = SMULLOHI->getOperand(1);
14418
14419 SelectionDAG &DAG = DCI.DAG;
14420 if (!isS16(OpS16, DAG) && !isSRA16(OpS16)) {
14421 OpS16 = OpS32;
14422 OpS32 = SMULLOHI->getOperand(0);
14423 }
14424
14425 SDLoc dl(OR);
14426 unsigned Opcode = 0;
14427 if (isS16(OpS16, DAG))
14428 Opcode = ARMISD::SMULWB;
14429 else if (isSRA16(OpS16)) {
14430 Opcode = ARMISD::SMULWT;
14431 OpS16 = OpS16->getOperand(0);
14432 }
14433 else
14434 return SDValue();
14435
14436 SDValue Res = DAG.getNode(Opcode, dl, MVT::i32, OpS32, OpS16);
14437 DAG.ReplaceAllUsesOfValueWith(SDValue(OR, 0), Res);
14438 return SDValue(OR, 0);
14439}
14440
14443 const ARMSubtarget *Subtarget) {
14444 // BFI is only available on V6T2+
14445 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops())
14446 return SDValue();
14447
14448 EVT VT = N->getValueType(0);
14449 SDValue N0 = N->getOperand(0);
14450 SDValue N1 = N->getOperand(1);
14451 SelectionDAG &DAG = DCI.DAG;
14452 SDLoc DL(N);
14453 // 1) or (and A, mask), val => ARMbfi A, val, mask
14454 // iff (val & mask) == val
14455 //
14456 // 2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14457 // 2a) iff isBitFieldInvertedMask(mask) && isBitFieldInvertedMask(~mask2)
14458 // && mask == ~mask2
14459 // 2b) iff isBitFieldInvertedMask(~mask) && isBitFieldInvertedMask(mask2)
14460 // && ~mask == mask2
14461 // (i.e., copy a bitfield value into another bitfield of the same width)
14462
14463 if (VT != MVT::i32)
14464 return SDValue();
14465
14466 SDValue N00 = N0.getOperand(0);
14467
14468 // The value and the mask need to be constants so we can verify this is
14469 // actually a bitfield set. If the mask is 0xffff, we can do better
14470 // via a movt instruction, so don't use BFI in that case.
14471 SDValue MaskOp = N0.getOperand(1);
14473 if (!MaskC)
14474 return SDValue();
14475 unsigned Mask = MaskC->getZExtValue();
14476 if (Mask == 0xffff)
14477 return SDValue();
14478 SDValue Res;
14479 // Case (1): or (and A, mask), val => ARMbfi A, val, mask
14481 if (N1C) {
14482 unsigned Val = N1C->getZExtValue();
14483 if ((Val & ~Mask) != Val)
14484 return SDValue();
14485
14486 if (ARM::isBitFieldInvertedMask(Mask)) {
14487 Val >>= llvm::countr_zero(~Mask);
14488
14489 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00,
14490 DAG.getConstant(Val, DL, MVT::i32),
14491 DAG.getConstant(Mask, DL, MVT::i32));
14492
14493 DCI.CombineTo(N, Res, false);
14494 // Return value from the original node to inform the combiner than N is
14495 // now dead.
14496 return SDValue(N, 0);
14497 }
14498 } else if (N1.getOpcode() == ISD::AND) {
14499 // case (2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14501 if (!N11C)
14502 return SDValue();
14503 unsigned Mask2 = N11C->getZExtValue();
14504
14505 // Mask and ~Mask2 (or reverse) must be equivalent for the BFI pattern
14506 // as is to match.
14507 if (ARM::isBitFieldInvertedMask(Mask) &&
14508 (Mask == ~Mask2)) {
14509 // The pack halfword instruction works better for masks that fit it,
14510 // so use that when it's available.
14511 if (Subtarget->hasDSP() &&
14512 (Mask == 0xffff || Mask == 0xffff0000))
14513 return SDValue();
14514 // 2a
14515 unsigned amt = llvm::countr_zero(Mask2);
14516 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0),
14517 DAG.getConstant(amt, DL, MVT::i32));
14518 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res,
14519 DAG.getConstant(Mask, DL, MVT::i32));
14520 DCI.CombineTo(N, Res, false);
14521 // Return value from the original node to inform the combiner than N is
14522 // now dead.
14523 return SDValue(N, 0);
14524 } else if (ARM::isBitFieldInvertedMask(~Mask) &&
14525 (~Mask == Mask2)) {
14526 // The pack halfword instruction works better for masks that fit it,
14527 // so use that when it's available.
14528 if (Subtarget->hasDSP() &&
14529 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14530 return SDValue();
14531 // 2b
14532 unsigned lsb = llvm::countr_zero(Mask);
14533 Res = DAG.getNode(ISD::SRL, DL, VT, N00,
14534 DAG.getConstant(lsb, DL, MVT::i32));
14535 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res,
14536 DAG.getConstant(Mask2, DL, MVT::i32));
14537 DCI.CombineTo(N, Res, false);
14538 // Return value from the original node to inform the combiner than N is
14539 // now dead.
14540 return SDValue(N, 0);
14541 }
14542 }
14543
14544 if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) &&
14545 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N00.getOperand(1)) &&
14547 // Case (3): or (and (shl A, #shamt), mask), B => ARMbfi B, A, ~mask
14548 // where lsb(mask) == #shamt and masked bits of B are known zero.
14549 SDValue ShAmt = N00.getOperand(1);
14550 unsigned ShAmtC = ShAmt->getAsZExtVal();
14551 unsigned LSB = llvm::countr_zero(Mask);
14552 if (ShAmtC != LSB)
14553 return SDValue();
14554
14555 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0),
14556 DAG.getConstant(~Mask, DL, MVT::i32));
14557
14558 DCI.CombineTo(N, Res, false);
14559 // Return value from the original node to inform the combiner than N is
14560 // now dead.
14561 return SDValue(N, 0);
14562 }
14563
14564 return SDValue();
14565}
14566
14567static bool isValidMVECond(unsigned CC, bool IsFloat) {
14568 switch (CC) {
14569 case ARMCC::EQ:
14570 case ARMCC::NE:
14571 case ARMCC::LE:
14572 case ARMCC::GT:
14573 case ARMCC::GE:
14574 case ARMCC::LT:
14575 return true;
14576 case ARMCC::HS:
14577 case ARMCC::HI:
14578 return !IsFloat;
14579 default:
14580 return false;
14581 };
14582}
14583
14585 if (N->getOpcode() == ARMISD::VCMP)
14586 return (ARMCC::CondCodes)N->getConstantOperandVal(2);
14587 else if (N->getOpcode() == ARMISD::VCMPZ)
14588 return (ARMCC::CondCodes)N->getConstantOperandVal(1);
14589 else
14590 llvm_unreachable("Not a VCMP/VCMPZ!");
14591}
14592
14595 return isValidMVECond(CC, N->getOperand(0).getValueType().isFloatingPoint());
14596}
14597
14599 const ARMSubtarget *Subtarget) {
14600 // Try to invert "or A, B" -> "and ~A, ~B", as the "and" is easier to chain
14601 // together with predicates
14602 EVT VT = N->getValueType(0);
14603 SDLoc DL(N);
14604 SDValue N0 = N->getOperand(0);
14605 SDValue N1 = N->getOperand(1);
14606
14607 auto IsFreelyInvertable = [&](SDValue V) {
14608 if (V->getOpcode() == ARMISD::VCMP || V->getOpcode() == ARMISD::VCMPZ)
14609 return CanInvertMVEVCMP(V);
14610 return false;
14611 };
14612
14613 // At least one operand must be freely invertable.
14614 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14615 return SDValue();
14616
14617 SDValue NewN0 = DAG.getLogicalNOT(DL, N0, VT);
14618 SDValue NewN1 = DAG.getLogicalNOT(DL, N1, VT);
14619 SDValue And = DAG.getNode(ISD::AND, DL, VT, NewN0, NewN1);
14620 return DAG.getLogicalNOT(DL, And, VT);
14621}
14622
14623// Try to form a NEON shift-{right, left}-and-insert (VSRI/VSLI) from:
14624// (or (and X, splat (i32 C1)), (srl Y, splat (i32 C2))) -> VSRI X, Y, #C2
14625// (or (and X, splat (i32 C1)), (shl Y, splat (i32 C2))) -> VSLI X, Y, #C2
14626// where C1 is a mask that preserves the bits not written by the shift/insert,
14627// i.e. `C1 == (1 << C2) - 1`.
14629 SDValue ShiftOp, EVT VT,
14630 SDLoc dl) {
14631 // Match (and X, Mask)
14632 if (AndOp.getOpcode() != ISD::AND)
14633 return SDValue();
14634
14635 SDValue X = AndOp.getOperand(0);
14636 SDValue Mask = AndOp.getOperand(1);
14637
14638 ConstantSDNode *MaskC = isConstOrConstSplat(Mask, false, true);
14639 if (!MaskC)
14640 return SDValue();
14641 APInt MaskBits =
14642 MaskC->getAPIntValue().trunc(Mask.getScalarValueSizeInBits());
14643
14644 // Match shift (srl/shl Y, CntVec)
14645 int64_t Cnt = 0;
14646 bool IsShiftRight = false;
14647 SDValue Y;
14648
14649 if (ShiftOp.getOpcode() == ARMISD::VSHRuIMM) {
14650 IsShiftRight = true;
14651 Y = ShiftOp.getOperand(0);
14652 Cnt = ShiftOp.getConstantOperandVal(1);
14653 } else if (ShiftOp.getOpcode() == ARMISD::VSHLIMM) {
14654 Y = ShiftOp.getOperand(0);
14655 Cnt = ShiftOp.getConstantOperandVal(1);
14656 } else {
14657 return SDValue();
14658 }
14659
14660 unsigned ElemBits = VT.getScalarSizeInBits();
14661 APInt RequiredMask = IsShiftRight
14662 ? APInt::getHighBitsSet(ElemBits, (unsigned)Cnt)
14663 : APInt::getLowBitsSet(ElemBits, (unsigned)Cnt);
14664 if (MaskBits != RequiredMask)
14665 return SDValue();
14666
14667 unsigned Opc = IsShiftRight ? ARMISD::VSRIIMM : ARMISD::VSLIIMM;
14668 return DAG.getNode(Opc, dl, VT, X, Y, DAG.getConstant(Cnt, dl, MVT::i32));
14669}
14670
14671/// PerformORCombine - Target-specific dag combine xforms for ISD::OR
14673 const ARMSubtarget *Subtarget) {
14674 // Attempt to use immediate-form VORR
14675 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
14676 SDLoc dl(N);
14677 EVT VT = N->getValueType(0);
14678 SelectionDAG &DAG = DCI.DAG;
14679
14680 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14681 return SDValue();
14682
14683 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14684 VT == MVT::v8i1 || VT == MVT::v16i1))
14685 return PerformORCombine_i1(N, DAG, Subtarget);
14686
14687 APInt SplatBits, SplatUndef;
14688 unsigned SplatBitSize;
14689 bool HasAnyUndefs;
14690 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14691 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14692 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14693 SplatBitSize == 64) {
14694 EVT VorrVT;
14695 SDValue Val =
14696 isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
14697 SplatBitSize, DAG, dl, VorrVT, VT, OtherModImm);
14698 if (Val.getNode()) {
14699 SDValue Input =
14700 DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VorrVT, N->getOperand(0));
14701 SDValue Vorr = DAG.getNode(ARMISD::VORRIMM, dl, VorrVT, Input, Val);
14702 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vorr);
14703 }
14704 }
14705 }
14706
14707 if (!Subtarget->isThumb1Only()) {
14708 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
14709 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
14710 return Result;
14711 if (SDValue Result = PerformORCombineToSMULWBT(N, DCI, Subtarget))
14712 return Result;
14713 }
14714
14715 SDValue N0 = N->getOperand(0);
14716 SDValue N1 = N->getOperand(1);
14717
14718 // (or (and X, C1), (srl Y, C2)) -> VSRI X, Y, #C2
14719 // (or (and X, C1), (shl Y, C2)) -> VSLI X, Y, #C2
14720 if (VT.isVector() &&
14721 ((Subtarget->hasNEON() && DAG.getTargetLoweringInfo().isTypeLegal(VT)) ||
14722 (Subtarget->hasMVEIntegerOps() &&
14723 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32)))) {
14724 if (SDValue ShiftInsert =
14725 PerformORCombineToShiftInsert(DAG, N0, N1, VT, dl))
14726 return ShiftInsert;
14727
14728 if (SDValue ShiftInsert =
14729 PerformORCombineToShiftInsert(DAG, N1, N0, VT, dl))
14730 return ShiftInsert;
14731 }
14732
14733 // (or (and B, A), (and C, ~A)) => (VBSL A, B, C) when A is a constant.
14734 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() &&
14736
14737 // The code below optimizes (or (and X, Y), Z).
14738 // The AND operand needs to have a single user to make these optimizations
14739 // profitable.
14740 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
14741 return SDValue();
14742
14743 APInt SplatUndef;
14744 unsigned SplatBitSize;
14745 bool HasAnyUndefs;
14746
14747 APInt SplatBits0, SplatBits1;
14750 // Ensure that the second operand of both ands are constants
14751 if (BVN0 && BVN0->isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14752 HasAnyUndefs) && !HasAnyUndefs) {
14753 if (BVN1 && BVN1->isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14754 HasAnyUndefs) && !HasAnyUndefs) {
14755 // Ensure that the bit width of the constants are the same and that
14756 // the splat arguments are logical inverses as per the pattern we
14757 // are trying to simplify.
14758 if (SplatBits0.getBitWidth() == SplatBits1.getBitWidth() &&
14759 SplatBits0 == ~SplatBits1) {
14760 // Canonicalize the vector type to make instruction selection
14761 // simpler.
14762 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
14763 SDValue Mask = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl,
14764 CanonicalVT, N0->getOperand(1));
14765 SDValue LHS = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl,
14766 CanonicalVT, N0->getOperand(0));
14767 SDValue RHS = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl,
14768 CanonicalVT, N1->getOperand(0));
14769 SDValue Result =
14770 DAG.getNode(ARMISD::VBSP, dl, CanonicalVT, Mask, LHS, RHS);
14771 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Result);
14772 }
14773 }
14774 }
14775 }
14776
14777 // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when
14778 // reasonable.
14779 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
14780 if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget))
14781 return Res;
14782 }
14783
14784 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
14785 return Result;
14786
14787 // (or x, (csinc 0, 0, cc)) -> (csinc x, 0, cc)
14788 // providing that the x is 0 or 1.
14789 SDValue CSINC = N1;
14790 SDValue Other = N0;
14791 if (CSINC.getOpcode() != ARMISD::CSINC)
14792 std::swap(CSINC, Other);
14793 if (CSINC.getOpcode() == ARMISD::CSINC &&
14794 isNullConstant(CSINC.getOperand(0)) &&
14795 isNullConstant(CSINC.getOperand(1)) &&
14797 return DAG.getNode(ARMISD::CSINC, dl, VT, Other, CSINC.getOperand(1),
14798 CSINC.getOperand(2), CSINC.getOperand(3));
14799
14800 return SDValue();
14801}
14802
14805 const ARMSubtarget *Subtarget) {
14806 EVT VT = N->getValueType(0);
14807 SelectionDAG &DAG = DCI.DAG;
14808
14809 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14810 return SDValue();
14811
14812 if (!Subtarget->isThumb1Only()) {
14813 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
14814 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
14815 return Result;
14816
14817 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
14818 return Result;
14819 }
14820
14821 if (Subtarget->hasMVEIntegerOps()) {
14822 // fold (xor(vcmp/z, 1)) into a vcmp with the opposite condition.
14823 SDValue N0 = N->getOperand(0);
14824 SDValue N1 = N->getOperand(1);
14825 const TargetLowering *TLI = Subtarget->getTargetLowering();
14826 if (TLI->isConstTrueVal(N1) &&
14827 (N0->getOpcode() == ARMISD::VCMP || N0->getOpcode() == ARMISD::VCMPZ)) {
14828 if (CanInvertMVEVCMP(N0)) {
14829 SDLoc DL(N0);
14831
14833 Ops.push_back(N0->getOperand(0));
14834 if (N0->getOpcode() == ARMISD::VCMP)
14835 Ops.push_back(N0->getOperand(1));
14836 Ops.push_back(DAG.getConstant(CC, DL, MVT::i32));
14837 return DAG.getNode(N0->getOpcode(), DL, N0->getValueType(0), Ops);
14838 }
14839 }
14840 }
14841
14842 return SDValue();
14843}
14844
14845// ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
14846// and fill in FromMask and ToMask with (consecutive) bits in "from" to be extracted and
14847// their position in "to" (Rd).
14848static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
14849 assert(N->getOpcode() == ARMISD::BFI);
14850
14851 SDValue From = N->getOperand(1);
14852 ToMask = ~N->getConstantOperandAPInt(2);
14853 FromMask = APInt::getLowBitsSet(ToMask.getBitWidth(), ToMask.popcount());
14854
14855 // If the Base came from a SHR #C, we can deduce that it is really testing bit
14856 // #C in the base of the SHR.
14857 if (From->getOpcode() == ISD::SRL &&
14858 isa<ConstantSDNode>(From->getOperand(1))) {
14859 APInt Shift = From->getConstantOperandAPInt(1);
14860 assert(Shift.getLimitedValue() < 32 && "Shift too large!");
14861 FromMask <<= Shift.getLimitedValue(31);
14862 From = From->getOperand(0);
14863 }
14864
14865 return From;
14866}
14867
14868// If A and B contain one contiguous set of bits, does A | B == A . B?
14869//
14870// Neither A nor B must be zero.
14871static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) {
14872 unsigned LastActiveBitInA = A.countr_zero();
14873 unsigned FirstActiveBitInB = B.getBitWidth() - B.countl_zero() - 1;
14874 return LastActiveBitInA - 1 == FirstActiveBitInB;
14875}
14876
14878 // We have a BFI in N. Find a BFI it can combine with, if one exists.
14879 APInt ToMask, FromMask;
14880 SDValue From = ParseBFI(N, ToMask, FromMask);
14881 SDValue To = N->getOperand(0);
14882
14883 SDValue V = To;
14884 if (V.getOpcode() != ARMISD::BFI)
14885 return SDValue();
14886
14887 APInt NewToMask, NewFromMask;
14888 SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask);
14889 if (NewFrom != From)
14890 return SDValue();
14891
14892 // Do the written bits conflict with any we've seen so far?
14893 if ((NewToMask & ToMask).getBoolValue())
14894 // Conflicting bits.
14895 return SDValue();
14896
14897 // Are the new bits contiguous when combined with the old bits?
14898 if (BitsProperlyConcatenate(ToMask, NewToMask) &&
14899 BitsProperlyConcatenate(FromMask, NewFromMask))
14900 return V;
14901 if (BitsProperlyConcatenate(NewToMask, ToMask) &&
14902 BitsProperlyConcatenate(NewFromMask, FromMask))
14903 return V;
14904
14905 return SDValue();
14906}
14907
14909 SDValue N0 = N->getOperand(0);
14910 SDValue N1 = N->getOperand(1);
14911
14912 if (N1.getOpcode() == ISD::AND) {
14913 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff
14914 // the bits being cleared by the AND are not demanded by the BFI.
14916 if (!N11C)
14917 return SDValue();
14918 unsigned InvMask = N->getConstantOperandVal(2);
14919 unsigned LSB = llvm::countr_zero(~InvMask);
14920 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB;
14921 assert(Width <
14922 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14923 "undefined behavior");
14924 unsigned Mask = (1u << Width) - 1;
14925 unsigned Mask2 = N11C->getZExtValue();
14926 if ((Mask & (~Mask2)) == 0)
14927 return DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0),
14928 N->getOperand(0), N1.getOperand(0), N->getOperand(2));
14929 return SDValue();
14930 }
14931
14932 // Look for another BFI to combine with.
14933 if (SDValue CombineBFI = FindBFIToCombineWith(N)) {
14934 // We've found a BFI.
14935 APInt ToMask1, FromMask1;
14936 SDValue From1 = ParseBFI(N, ToMask1, FromMask1);
14937
14938 APInt ToMask2, FromMask2;
14939 SDValue From2 = ParseBFI(CombineBFI.getNode(), ToMask2, FromMask2);
14940 assert(From1 == From2);
14941 (void)From2;
14942
14943 // Create a new BFI, combining the two together.
14944 APInt NewFromMask = FromMask1 | FromMask2;
14945 APInt NewToMask = ToMask1 | ToMask2;
14946
14947 EVT VT = N->getValueType(0);
14948 SDLoc dl(N);
14949
14950 if (NewFromMask[0] == 0)
14951 From1 = DAG.getNode(ISD::SRL, dl, VT, From1,
14952 DAG.getConstant(NewFromMask.countr_zero(), dl, VT));
14953 return DAG.getNode(ARMISD::BFI, dl, VT, CombineBFI.getOperand(0), From1,
14954 DAG.getConstant(~NewToMask, dl, VT));
14955 }
14956
14957 // Reassociate BFI(BFI (A, B, M1), C, M2) to BFI(BFI (A, C, M2), B, M1) so
14958 // that lower bit insertions are performed first, providing that M1 and M2
14959 // do no overlap. This can allow multiple BFI instructions to be combined
14960 // together by the other folds above.
14961 if (N->getOperand(0).getOpcode() == ARMISD::BFI) {
14962 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
14963 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14964
14965 if (!N0.hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14966 ToMask1.countl_zero() < ToMask2.countl_zero())
14967 return SDValue();
14968
14969 EVT VT = N->getValueType(0);
14970 SDLoc dl(N);
14971 SDValue BFI1 = DAG.getNode(ARMISD::BFI, dl, VT, N0.getOperand(0),
14972 N->getOperand(1), N->getOperand(2));
14973 return DAG.getNode(ARMISD::BFI, dl, VT, BFI1, N0.getOperand(1),
14974 N0.getOperand(2));
14975 }
14976
14977 return SDValue();
14978}
14979
14980// Check that N is CMPZ(CSINC(0, 0, CC, X)),
14981// or CMPZ(CMOV(1, 0, CC, X))
14982// return X if valid.
14984 if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(Cmp->getOperand(1)))
14985 return SDValue();
14986 SDValue CSInc = Cmp->getOperand(0);
14987
14988 // Ignore any `And 1` nodes that may not yet have been removed. We are
14989 // looking for a value that produces 1/0, so these have no effect on the
14990 // code.
14991 while (CSInc.getOpcode() == ISD::AND &&
14992 isa<ConstantSDNode>(CSInc.getOperand(1)) &&
14993 CSInc.getConstantOperandVal(1) == 1 && CSInc->hasOneUse())
14994 CSInc = CSInc.getOperand(0);
14995
14996 if (CSInc.getOpcode() == ARMISD::CSINC &&
14997 isNullConstant(CSInc.getOperand(0)) &&
14998 isNullConstant(CSInc.getOperand(1)) && CSInc->hasOneUse()) {
15000 return CSInc.getOperand(3);
15001 }
15002 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(CSInc.getOperand(0)) &&
15003 isNullConstant(CSInc.getOperand(1)) && CSInc->hasOneUse()) {
15005 return CSInc.getOperand(3);
15006 }
15007 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(CSInc.getOperand(1)) &&
15008 isNullConstant(CSInc.getOperand(0)) && CSInc->hasOneUse()) {
15011 return CSInc.getOperand(3);
15012 }
15013 return SDValue();
15014}
15015
15017 // Given CMPZ(CSINC(C, 0, 0, EQ), 0), we can just use C directly. As in
15018 // t92: flags = ARMISD::CMPZ t74, 0
15019 // t93: i32 = ARMISD::CSINC 0, 0, 1, t92
15020 // t96: flags = ARMISD::CMPZ t93, 0
15021 // t114: i32 = ARMISD::CSINV 0, 0, 0, t96
15023 if (SDValue C = IsCMPZCSINC(N, Cond))
15024 if (Cond == ARMCC::EQ)
15025 return C;
15026 return SDValue();
15027}
15028
15030 // Fold away an unnecessary CMPZ/CSINC
15031 // CSXYZ A, B, C1 (CMPZ (CSINC 0, 0, C2, D), 0) ->
15032 // if C1==EQ -> CSXYZ A, B, C2, D
15033 // if C1==NE -> CSXYZ A, B, NOT(C2), D
15035 if (SDValue C = IsCMPZCSINC(N->getOperand(3).getNode(), Cond)) {
15036 if (N->getConstantOperandVal(2) == ARMCC::EQ)
15037 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
15038 N->getOperand(1),
15039 DAG.getConstant(Cond, SDLoc(N), MVT::i32), C);
15040 if (N->getConstantOperandVal(2) == ARMCC::NE)
15041 return DAG.getNode(
15042 N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
15043 N->getOperand(1),
15045 }
15046 return SDValue();
15047}
15048
15049/// PerformVMOVRRDCombine - Target-specific dag combine xforms for
15050/// ARMISD::VMOVRRD.
15053 const ARMSubtarget *Subtarget) {
15054 // vmovrrd(vmovdrr x, y) -> x,y
15055 SDValue InDouble = N->getOperand(0);
15056 if (InDouble.getOpcode() == ARMISD::VMOVDRR && Subtarget->hasFP64())
15057 return DCI.CombineTo(N, InDouble.getOperand(0), InDouble.getOperand(1));
15058
15059 // vmovrrd(load f64) -> (load i32), (load i32)
15060 SDNode *InNode = InDouble.getNode();
15061 if (ISD::isNormalLoad(InNode) && InNode->hasOneUse() &&
15062 InNode->getValueType(0) == MVT::f64 &&
15063 InNode->getOperand(1).getOpcode() == ISD::FrameIndex &&
15064 !cast<LoadSDNode>(InNode)->isVolatile()) {
15065 // TODO: Should this be done for non-FrameIndex operands?
15066 LoadSDNode *LD = cast<LoadSDNode>(InNode);
15067
15068 SelectionDAG &DAG = DCI.DAG;
15069 SDLoc DL(LD);
15070 SDValue BasePtr = LD->getBasePtr();
15071 SDValue NewLD1 =
15072 DAG.getLoad(MVT::i32, DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15073 LD->getAlign(), LD->getMemOperand()->getFlags());
15074
15075 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
15076 DAG.getConstant(4, DL, MVT::i32));
15077
15078 SDValue NewLD2 = DAG.getLoad(MVT::i32, DL, LD->getChain(), OffsetPtr,
15079 LD->getPointerInfo().getWithOffset(4),
15080 commonAlignment(LD->getAlign(), 4),
15081 LD->getMemOperand()->getFlags());
15082
15083 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLD2.getValue(1));
15084 if (DCI.DAG.getDataLayout().isBigEndian())
15085 std::swap (NewLD1, NewLD2);
15086 SDValue Result = DCI.CombineTo(N, NewLD1, NewLD2);
15087 return Result;
15088 }
15089
15090 // VMOVRRD(extract(..(build_vector(a, b, c, d)))) -> a,b or c,d
15091 // VMOVRRD(extract(insert_vector(insert_vector(.., a, l1), b, l2))) -> a,b
15092 if (InDouble.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15093 isa<ConstantSDNode>(InDouble.getOperand(1))) {
15094 SDValue BV = InDouble.getOperand(0);
15095 // Look up through any nop bitcasts and vector_reg_casts. bitcasts may
15096 // change lane order under big endian.
15097 bool BVSwap = BV.getOpcode() == ISD::BITCAST;
15098 while (
15099 (BV.getOpcode() == ISD::BITCAST ||
15100 BV.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
15101 (BV.getValueType() == MVT::v2f64 || BV.getValueType() == MVT::v2i64)) {
15102 BVSwap = BV.getOpcode() == ISD::BITCAST;
15103 BV = BV.getOperand(0);
15104 }
15105 if (BV.getValueType() != MVT::v4i32)
15106 return SDValue();
15107
15108 // Handle buildvectors, pulling out the correct lane depending on
15109 // endianness.
15110 unsigned Offset = InDouble.getConstantOperandVal(1) == 1 ? 2 : 0;
15111 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
15112 SDValue Op0 = BV.getOperand(Offset);
15113 SDValue Op1 = BV.getOperand(Offset + 1);
15114 if (!Subtarget->isLittle() && BVSwap)
15115 std::swap(Op0, Op1);
15116
15117 return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N));
15118 }
15119
15120 // A chain of insert_vectors, grabbing the correct value of the chain of
15121 // inserts.
15122 SDValue Op0, Op1;
15123 while (BV.getOpcode() == ISD::INSERT_VECTOR_ELT) {
15124 if (isa<ConstantSDNode>(BV.getOperand(2))) {
15125 if (BV.getConstantOperandVal(2) == Offset && !Op0)
15126 Op0 = BV.getOperand(1);
15127 if (BV.getConstantOperandVal(2) == Offset + 1 && !Op1)
15128 Op1 = BV.getOperand(1);
15129 }
15130 BV = BV.getOperand(0);
15131 }
15132 if (!Subtarget->isLittle() && BVSwap)
15133 std::swap(Op0, Op1);
15134 if (Op0 && Op1)
15135 return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N));
15136 }
15137
15138 return SDValue();
15139}
15140
15141/// PerformVMOVDRRCombine - Target-specific dag combine xforms for
15142/// ARMISD::VMOVDRR. This is also used for BUILD_VECTORs with 2 operands.
15144 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X)
15145 SDValue Op0 = N->getOperand(0);
15146 SDValue Op1 = N->getOperand(1);
15147 if (Op0.getOpcode() == ISD::BITCAST)
15148 Op0 = Op0.getOperand(0);
15149 if (Op1.getOpcode() == ISD::BITCAST)
15150 Op1 = Op1.getOperand(0);
15151 if (Op0.getOpcode() == ARMISD::VMOVRRD &&
15152 Op0.getNode() == Op1.getNode() &&
15153 Op0.getResNo() == 0 && Op1.getResNo() == 1)
15154 return DAG.getNode(ISD::BITCAST, SDLoc(N),
15155 N->getValueType(0), Op0.getOperand(0));
15156 return SDValue();
15157}
15158
15161 SDValue Op0 = N->getOperand(0);
15162
15163 // VMOVhr (VMOVrh (X)) -> X
15164 if (Op0->getOpcode() == ARMISD::VMOVrh)
15165 return Op0->getOperand(0);
15166
15167 // FullFP16: half values are passed in S-registers, and we don't
15168 // need any of the bitcast and moves:
15169 //
15170 // t2: f32,ch1,gl1? = CopyFromReg ch, Register:f32 %0, gl?
15171 // t5: i32 = bitcast t2
15172 // t18: f16 = ARMISD::VMOVhr t5
15173 // =>
15174 // tN: f16,ch2,gl2? = CopyFromReg ch, Register::f32 %0, gl?
15175 if (Op0->getOpcode() == ISD::BITCAST) {
15176 SDValue Copy = Op0->getOperand(0);
15177 if (Copy.getValueType() == MVT::f32 &&
15178 Copy->getOpcode() == ISD::CopyFromReg) {
15179 bool HasGlue = Copy->getNumOperands() == 3;
15180 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15181 HasGlue ? Copy->getOperand(2) : SDValue()};
15182 EVT OutTys[] = {N->getValueType(0), MVT::Other, MVT::Glue};
15183 SDValue NewCopy =
15185 DCI.DAG.getVTList(ArrayRef(OutTys, HasGlue ? 3 : 2)),
15186 ArrayRef(Ops, HasGlue ? 3 : 2));
15187
15188 // Update Users, Chains, and Potential Glue.
15189 DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), NewCopy.getValue(0));
15190 DCI.DAG.ReplaceAllUsesOfValueWith(Copy.getValue(1), NewCopy.getValue(1));
15191 if (HasGlue)
15192 DCI.DAG.ReplaceAllUsesOfValueWith(Copy.getValue(2),
15193 NewCopy.getValue(2));
15194
15195 return NewCopy;
15196 }
15197 }
15198
15199 // fold (VMOVhr (load x)) -> (load (f16*)x)
15200 if (LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
15201 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15202 LN0->getMemoryVT() == MVT::i16) {
15203 SDValue Load =
15204 DCI.DAG.getLoad(N->getValueType(0), SDLoc(N), LN0->getChain(),
15205 LN0->getBasePtr(), LN0->getMemOperand());
15206 DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
15207 DCI.DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Load.getValue(1));
15208 return Load;
15209 }
15210 }
15211
15212 // Only the bottom 16 bits of the source register are used.
15213 APInt DemandedMask = APInt::getLowBitsSet(32, 16);
15215 if (TLI.SimplifyDemandedBits(Op0, DemandedMask, DCI))
15216 return SDValue(N, 0);
15217
15218 return SDValue();
15219}
15220
15222 SDValue N0 = N->getOperand(0);
15223 EVT VT = N->getValueType(0);
15224
15225 // fold (VMOVrh (fpconst x)) -> const x
15227 APFloat V = C->getValueAPF();
15228 return DAG.getConstant(V.bitcastToAPInt().getZExtValue(), SDLoc(N), VT);
15229 }
15230
15231 // fold (VMOVrh (load x)) -> (zextload (i16*)x)
15232 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse()) {
15233 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
15234
15235 SDValue Load =
15236 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, LN0->getChain(),
15237 LN0->getBasePtr(), MVT::i16, LN0->getMemOperand());
15238 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
15239 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
15240 return Load;
15241 }
15242
15243 // Fold VMOVrh(extract(x, n)) -> vgetlaneu(x, n)
15244 if (N0->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15246 return DAG.getNode(ARMISD::VGETLANEu, SDLoc(N), VT, N0->getOperand(0),
15247 N0->getOperand(1));
15248
15249 return SDValue();
15250}
15251
15253 SDValue Op0 = N->getOperand(0);
15254 if (Op0->getOpcode() == ARMISD::VMOVsr)
15255 return Op0->getOperand(0);
15256 return SDValue();
15257}
15258
15259/// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
15260/// are normal, non-volatile loads. If so, it is profitable to bitcast an
15261/// i64 vector to have f64 elements, since the value can then be loaded
15262/// directly into a VFP register.
15264 unsigned NumElts = N->getValueType(0).getVectorNumElements();
15265 for (unsigned i = 0; i < NumElts; ++i) {
15266 SDNode *Elt = N->getOperand(i).getNode();
15267 if (ISD::isNormalLoad(Elt) && !cast<LoadSDNode>(Elt)->isVolatile())
15268 return true;
15269 }
15270 return false;
15271}
15272
15273/// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
15274/// ISD::BUILD_VECTOR.
15277 const ARMSubtarget *Subtarget) {
15278 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X):
15279 // VMOVRRD is introduced when legalizing i64 types. It forces the i64 value
15280 // into a pair of GPRs, which is fine when the value is used as a scalar,
15281 // but if the i64 value is converted to a vector, we need to undo the VMOVRRD.
15282 SelectionDAG &DAG = DCI.DAG;
15283 if (N->getNumOperands() == 2)
15284 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
15285 return RV;
15286
15287 // Load i64 elements as f64 values so that type legalization does not split
15288 // them up into i32 values.
15289 EVT VT = N->getValueType(0);
15290 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N))
15291 return SDValue();
15292 SDLoc dl(N);
15294 unsigned NumElts = VT.getVectorNumElements();
15295 for (unsigned i = 0; i < NumElts; ++i) {
15296 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i));
15297 Ops.push_back(V);
15298 // Make the DAGCombiner fold the bitcast.
15299 DCI.AddToWorklist(V.getNode());
15300 }
15301 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, NumElts);
15302 SDValue BV = DAG.getBuildVector(FloatVT, dl, Ops);
15303 return DAG.getNode(ISD::BITCAST, dl, VT, BV);
15304}
15305
15308 const ARMSubtarget *Subtarget) {
15309 // Type legalization splits a wide load into consecutive legal loads. Combine
15310 // each group used by a legal VECTOR_DEINTERLEAVE into a structured load.
15312 return SDValue();
15313
15314 unsigned NumParts = N->getNumOperands();
15316 return SDValue();
15317
15318 if (NumParts == 3 && Subtarget->hasMVEIntegerOps())
15319 return SDValue();
15320
15321 EVT SubVecTy = N->getValueType(0);
15322 if (Subtarget->hasNEON() && !SubVecTy.is64BitVector() &&
15323 !SubVecTy.is128BitVector())
15324 return SDValue();
15325 if (Subtarget->hasMVEIntegerOps() && !SubVecTy.is128BitVector())
15326 return SDValue();
15327 unsigned EltBits = SubVecTy.getScalarSizeInBits();
15328 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
15329 return SDValue();
15330
15332 for (const SDValue &Operand : N->op_values()) {
15333 auto *Load = dyn_cast<LoadSDNode>(Operand);
15334 if (!Load || !Operand.hasOneUse())
15335 return SDValue();
15336 Loads.push_back(Load);
15337 }
15338
15339 LoadSDNode *BaseLoad = Loads[0];
15340 if (Subtarget->hasMVEIntegerOps() && BaseLoad->getAlign() < EltBits / 8)
15341 return SDValue();
15342 unsigned Bytes = SubVecTy.getStoreSize().getFixedValue();
15343 for (auto [Idx, Load] : enumerate(Loads)) {
15344 if (!DAG.areNonVolatileConsecutiveLoads(Load, BaseLoad, Bytes, Idx))
15345 return SDValue();
15346 }
15347
15348 SDLoc DL(N);
15349 EVT MemVT =
15350 EVT::getVectorVT(*DAG.getContext(), SubVecTy.getVectorElementType(),
15351 SubVecTy.getVectorElementCount() * NumParts);
15353 MachineMemOperand *MMO =
15354 MF.getMachineMemOperand(BaseLoad->getMemOperand(), 0, NumParts * Bytes);
15355
15357 ResVTs.push_back(MVT::Other);
15358
15359 SmallVector<SDValue> NewLdOps;
15360 NewLdOps.push_back(BaseLoad->getChain());
15361 Intrinsic::ID IID = 0;
15362 static constexpr Intrinsic::ID NEONLoads[] = {Intrinsic::arm_neon_vld2,
15363 Intrinsic::arm_neon_vld3,
15364 Intrinsic::arm_neon_vld4};
15365 if (Subtarget->hasNEON())
15366 IID = NEONLoads[NumParts - 2];
15367 else
15368 IID = NumParts == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
15369
15370 NewLdOps.push_back(DAG.getTargetConstant(IID, DL, MVT::i64));
15371 NewLdOps.push_back(BaseLoad->getBasePtr());
15372 // We can now generate a structured load!
15373 SDValue NewLoad = DAG.getMemIntrinsicNode(
15374 ISD::INTRINSIC_W_CHAIN, DL, DAG.getVTList(ResVTs), NewLdOps, MemVT, MMO);
15375
15377 for (unsigned I = 0; I != NumParts; ++I)
15378 ResOps.push_back(NewLoad.getValue(I));
15379
15380 // Preserve the memory ordering of each original load with the new load.
15381 for (LoadSDNode *Load : Loads)
15383 NewLoad.getValue(NumParts));
15384 return DCI.CombineTo(N, ResOps, false);
15385}
15386
15387// Combine a deinterleave of adjacent subvectors from one load into a
15388// structured NEON load.
15389static SDValue
15392 const ARMSubtarget *Subtarget) {
15393 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
15394 return SDValue();
15395
15397 Subtarget))
15398 return Res;
15399
15400 if (!DCI.isBeforeLegalize())
15401 return SDValue();
15402
15403 SelectionDAG &DAG = DCI.DAG;
15404 unsigned NumParts = N->getNumOperands();
15405 if (NumParts != 2 && NumParts != 3 && NumParts != 4)
15406 return SDValue();
15407
15408 if (NumParts == 3 && Subtarget->hasMVEIntegerOps())
15409 return SDValue();
15410
15411 EVT SubVecTy = N->getValueType(0);
15412
15413 if (Subtarget->hasNEON() && !SubVecTy.is64BitVector() &&
15414 !SubVecTy.is128BitVector())
15415 return SDValue();
15416 if (Subtarget->hasMVEIntegerOps() && !SubVecTy.is128BitVector())
15417 return SDValue();
15418 unsigned EltBits = SubVecTy.getScalarSizeInBits();
15419 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
15420 return SDValue();
15421
15422 // Make sure each input operand is the correct extract_subvector of the same
15423 // wider vector.
15424 unsigned MinNumElements = SubVecTy.getVectorMinNumElements();
15425 SDValue Op0 = N->getOperand(0);
15426 for (unsigned I = 0; I < NumParts; I++) {
15427 SDValue OpI = N->getOperand(I);
15428 if (!OpI.hasOneUse() || OpI.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
15429 OpI.getOperand(0) != Op0.getOperand(0))
15430 return SDValue();
15431 if (OpI.getConstantOperandVal(1) != (I * MinNumElements))
15432 return SDValue();
15433 }
15434
15435 SDValue WideVec = Op0.getOperand(0);
15436 SDLoc DL(N);
15437
15439 ResVTs.push_back(MVT::Other);
15440 SDVTList ResVTList = DAG.getVTList(ResVTs);
15441 auto *Load = dyn_cast<LoadSDNode>(WideVec);
15442 if (!Load || !Load->hasNUsesOfValue(NumParts, 0) || !Load->isSimple() ||
15443 !ISD::isNormalLoad(Load) || !Load->getOffset().isUndef())
15444 return SDValue();
15445 if (Subtarget->hasMVEIntegerOps() && Load->getAlign() < EltBits / 8)
15446 return SDValue();
15447
15448 static constexpr Intrinsic::ID NEONLoads[] = {Intrinsic::arm_neon_vld2,
15449 Intrinsic::arm_neon_vld3,
15450 Intrinsic::arm_neon_vld4};
15451 SmallVector<SDValue> NewLdOps;
15452 Intrinsic::ID IID = 0;
15453 NewLdOps.push_back(Load->getChain());
15454 if (Subtarget->hasNEON())
15455 IID = NEONLoads[NumParts - 2];
15456 else
15457 IID = NumParts == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
15458 NewLdOps.push_back(DAG.getTargetConstant(IID, DL, MVT::i64));
15459
15460 NewLdOps.push_back(Load->getBasePtr());
15461
15462 // We can now generate a structured load!
15463 SDValue Res =
15464 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, ResVTList, NewLdOps,
15465 Load->getMemoryVT(), Load->getMemOperand());
15466
15468 for (unsigned Idx = 0; Idx < NumParts; Idx++)
15469 ResOps[Idx] = Res.getValue(Idx);
15470
15471 // Preserve the original load's memory ordering with the new load.
15473 return DCI.CombineTo(N, ResOps, false);
15474}
15475
15476/// Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
15477static SDValue
15479 SelectionDAG &DAG = DCI.DAG;
15480 EVT VT = N->getValueType(0);
15481
15482 // Fold buildvector(extractelt(bitcast(build_vector(a,b,_,_))),
15483 // extractelt(bitcast(build_vector(_,_,c,d))))
15484 // -> bitcast(build_vector(a,b,c,d))
15485 if (VT.getScalarSizeInBits() == 64 && N->getNumOperands() == 2) {
15486 auto IsExtractFromBuildVec = [](SDValue V, SDValue &BV, int64_t &Lane) {
15487 if (!sd_match(V, m_ExtractElt(m_Value(BV), m_ConstInt(Lane))))
15488 return false;
15489 while (BV.getOpcode() == ISD::BITCAST)
15490 BV = BV.getOperand(0);
15491 return BV.getOpcode() == ARMISD::BUILD_VECTOR &&
15492 BV.getNumOperands() == 4 &&
15493 BV.getValueType().getScalarSizeInBits() == 32;
15494 };
15495 SDValue BV1, BV2;
15496 int64_t Lane1, Lane2;
15497 if (IsExtractFromBuildVec(N->getOperand(0), BV1, Lane1) &&
15498 IsExtractFromBuildVec(N->getOperand(1), BV2, Lane2))
15499 return DAG.getBitcast(
15500 VT, DAG.getNode(
15501 ARMISD::BUILD_VECTOR, N, MVT::v4f32,
15502 DAG.getBitcast(MVT::f32, BV1->getOperand(Lane1 * 2 + 0)),
15503 DAG.getBitcast(MVT::f32, BV1->getOperand(Lane1 * 2 + 1)),
15504 DAG.getBitcast(MVT::f32, BV2->getOperand(Lane2 * 2 + 0)),
15505 DAG.getBitcast(MVT::f32, BV2->getOperand(Lane2 * 2 + 1))));
15506 }
15507
15508 // ARMISD::BUILD_VECTOR is introduced when legalizing ISD::BUILD_VECTOR.
15509 // At that time, we may have inserted bitcasts from integer to float.
15510 // If these bitcasts have survived DAGCombine, change the lowering of this
15511 // BUILD_VECTOR in something more vector friendly, i.e., that does not
15512 // force to use floating point types.
15513
15514 // Make sure we can change the type of the vector.
15515 // This is possible iff:
15516 // 1. The vector is only used in a bitcast to a integer type. I.e.,
15517 // 1.1. Vector is used only once.
15518 // 1.2. Use is a bit convert to an integer type.
15519 // 2. The size of its operands are 32-bits (64-bits are not legal).
15520 EVT EltVT = VT.getVectorElementType();
15521
15522 // Check 1.1. and 2.
15523 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse())
15524 return SDValue();
15525
15526 // By construction, the input type must be float.
15527 assert(EltVT == MVT::f32 && "Unexpected type!");
15528
15529 // Check 1.2.
15530 SDNode *Use = *N->user_begin();
15531 if (Use->getOpcode() != ISD::BITCAST ||
15532 Use->getValueType(0).isFloatingPoint())
15533 return SDValue();
15534
15535 // Check profitability.
15536 // Model is, if more than half of the relevant operands are bitcast from
15537 // i32, turn the build_vector into a sequence of insert_vector_elt.
15538 // Relevant operands are everything that is not statically
15539 // (i.e., at compile time) bitcasted.
15540 unsigned NumOfBitCastedElts = 0;
15541 unsigned NumElts = VT.getVectorNumElements();
15542 unsigned NumOfRelevantElts = NumElts;
15543 for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
15544 SDValue Elt = N->getOperand(Idx);
15545 if (Elt->getOpcode() == ISD::BITCAST) {
15546 // Assume only bit cast to i32 will go away.
15547 if (Elt->getOperand(0).getValueType() == MVT::i32)
15548 ++NumOfBitCastedElts;
15549 } else if (Elt.isUndef() || isa<ConstantSDNode>(Elt))
15550 // Constants are statically casted, thus do not count them as
15551 // relevant operands.
15552 --NumOfRelevantElts;
15553 }
15554
15555 // Check if more than half of the elements require a non-free bitcast.
15556 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15557 return SDValue();
15558
15559 // Create the new vector type.
15560 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
15561 // Check if the type is legal.
15563 if (!TLI.isTypeLegal(VecVT))
15564 return SDValue();
15565
15566 // Combine:
15567 // ARMISD::BUILD_VECTOR E1, E2, ..., EN.
15568 // => BITCAST INSERT_VECTOR_ELT
15569 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1),
15570 // (BITCAST EN), N.
15571 SDValue Vec = DAG.getUNDEF(VecVT);
15572 SDLoc dl(N);
15573 for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
15574 SDValue V = N->getOperand(Idx);
15575 if (V.isUndef())
15576 continue;
15577 if (V.getOpcode() == ISD::BITCAST &&
15578 V->getOperand(0).getValueType() == MVT::i32)
15579 // Fold obvious case.
15580 V = V.getOperand(0);
15581 else {
15582 V = DAG.getNode(ISD::BITCAST, SDLoc(V), MVT::i32, V);
15583 // Make the DAGCombiner fold the bitcasts.
15584 DCI.AddToWorklist(V.getNode());
15585 }
15586 SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32);
15587 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx);
15588 }
15589 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec);
15590 // Make the DAGCombiner fold the bitcasts.
15591 DCI.AddToWorklist(Vec.getNode());
15592 return Vec;
15593}
15594
15595static SDValue
15597 EVT VT = N->getValueType(0);
15598 SDValue Op = N->getOperand(0);
15599 SDLoc dl(N);
15600
15601 // PREDICATE_CAST(PREDICATE_CAST(x)) == PREDICATE_CAST(x)
15602 if (Op->getOpcode() == ARMISD::PREDICATE_CAST) {
15603 // If the valuetypes are the same, we can remove the cast entirely.
15604 if (Op->getOperand(0).getValueType() == VT)
15605 return Op->getOperand(0);
15606 return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
15607 }
15608
15609 // Turn pred_cast(xor x, -1) into xor(pred_cast x, -1), in order to produce
15610 // more VPNOT which might get folded as else predicates.
15611 if (Op.getValueType() == MVT::i32 && isBitwiseNot(Op)) {
15612 SDValue X =
15613 DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
15614 SDValue C = DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
15615 DCI.DAG.getConstant(65535, dl, MVT::i32));
15616 return DCI.DAG.getNode(ISD::XOR, dl, VT, X, C);
15617 }
15618
15619 // Only the bottom 16 bits of the source register are used.
15620 if (Op.getValueType() == MVT::i32) {
15621 APInt DemandedMask = APInt::getLowBitsSet(32, 16);
15623 if (TLI.SimplifyDemandedBits(Op, DemandedMask, DCI))
15624 return SDValue(N, 0);
15625 }
15626 return SDValue();
15627}
15628
15630 const ARMSubtarget *ST) {
15631 EVT VT = N->getValueType(0);
15632 SDValue Op = N->getOperand(0);
15633 SDLoc dl(N);
15634
15635 // Under Little endian, a VECTOR_REG_CAST is equivalent to a BITCAST
15636 if (ST->isLittle())
15637 return DAG.getNode(ISD::BITCAST, dl, VT, Op);
15638
15639 // VT VECTOR_REG_CAST (VT Op) -> Op
15640 if (Op.getValueType() == VT)
15641 return Op;
15642 // VECTOR_REG_CAST undef -> undef
15643 if (Op.isUndef())
15644 return DAG.getUNDEF(VT);
15645
15646 // VECTOR_REG_CAST(VECTOR_REG_CAST(x)) == VECTOR_REG_CAST(x)
15647 if (Op->getOpcode() == ARMISD::VECTOR_REG_CAST) {
15648 // If the valuetypes are the same, we can remove the cast entirely.
15649 if (Op->getOperand(0).getValueType() == VT)
15650 return Op->getOperand(0);
15651 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Op->getOperand(0));
15652 }
15653
15654 return SDValue();
15655}
15656
15658 const ARMSubtarget *Subtarget) {
15659 if (!Subtarget->hasMVEIntegerOps())
15660 return SDValue();
15661
15662 EVT VT = N->getValueType(0);
15663 SDValue Op0 = N->getOperand(0);
15664 SDValue Op1 = N->getOperand(1);
15665 ARMCC::CondCodes Cond = (ARMCC::CondCodes)N->getConstantOperandVal(2);
15666 SDLoc dl(N);
15667
15668 // vcmp X, 0, cc -> vcmpz X, cc
15669 if (isZeroVector(Op1))
15670 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op0, N->getOperand(2));
15671
15672 unsigned SwappedCond = getSwappedCondition(Cond);
15673 if (isValidMVECond(SwappedCond, VT.isFloatingPoint())) {
15674 // vcmp 0, X, cc -> vcmpz X, reversed(cc)
15675 if (isZeroVector(Op0))
15676 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op1,
15677 DAG.getConstant(SwappedCond, dl, MVT::i32));
15678 // vcmp vdup(Y), X, cc -> vcmp X, vdup(Y), reversed(cc)
15679 if (Op0->getOpcode() == ARMISD::VDUP && Op1->getOpcode() != ARMISD::VDUP)
15680 return DAG.getNode(ARMISD::VCMP, dl, VT, Op1, Op0,
15681 DAG.getConstant(SwappedCond, dl, MVT::i32));
15682 }
15683
15684 return SDValue();
15685}
15686
15687/// PerformInsertEltCombine - Target-specific dag combine xforms for
15688/// ISD::INSERT_VECTOR_ELT.
15691 // Bitcast an i64 load inserted into a vector to f64.
15692 // Otherwise, the i64 value will be legalized to a pair of i32 values.
15693 EVT VT = N->getValueType(0);
15694 SDNode *Elt = N->getOperand(1).getNode();
15695 if (VT.getVectorElementType() != MVT::i64 ||
15696 !ISD::isNormalLoad(Elt) || cast<LoadSDNode>(Elt)->isVolatile())
15697 return SDValue();
15698
15699 SelectionDAG &DAG = DCI.DAG;
15700 SDLoc dl(N);
15701 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
15703 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0));
15704 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1));
15705 // Make the DAGCombiner fold the bitcasts.
15706 DCI.AddToWorklist(Vec.getNode());
15707 DCI.AddToWorklist(V.getNode());
15708 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT,
15709 Vec, V, N->getOperand(2));
15710 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt);
15711}
15712
15713// Convert a pair of extracts from the same base vector to a VMOVRRD. Either
15714// directly or bitcast to an integer if the original is a float vector.
15715// extract(x, n); extract(x, n+1) -> VMOVRRD(extract v2f64 x, n/2)
15716// bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD(extract x, n/2)
15717static SDValue
15719 EVT VT = N->getValueType(0);
15720 SDLoc dl(N);
15721
15722 if (!DCI.isAfterLegalizeDAG() || VT != MVT::i32 ||
15723 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(MVT::f64))
15724 return SDValue();
15725
15726 SDValue Ext = SDValue(N, 0);
15727 if (Ext.getOpcode() == ISD::BITCAST &&
15728 Ext.getOperand(0).getValueType() == MVT::f32)
15729 Ext = Ext.getOperand(0);
15730 if (Ext.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
15732 Ext.getConstantOperandVal(1) % 2 != 0)
15733 return SDValue();
15734 if (Ext->hasOneUse() && (Ext->user_begin()->getOpcode() == ISD::SINT_TO_FP ||
15735 Ext->user_begin()->getOpcode() == ISD::UINT_TO_FP))
15736 return SDValue();
15737
15738 SDValue Op0 = Ext.getOperand(0);
15739 EVT VecVT = Op0.getValueType();
15740 unsigned ResNo = Op0.getResNo();
15741 unsigned Lane = Ext.getConstantOperandVal(1);
15742 if (VecVT.getVectorNumElements() != 4)
15743 return SDValue();
15744
15745 // Find another extract, of Lane + 1
15746 auto OtherIt = find_if(Op0->users(), [&](SDNode *V) {
15747 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15748 isa<ConstantSDNode>(V->getOperand(1)) &&
15749 V->getConstantOperandVal(1) == Lane + 1 &&
15750 V->getOperand(0).getResNo() == ResNo;
15751 });
15752 if (OtherIt == Op0->users().end())
15753 return SDValue();
15754
15755 // For float extracts, we need to be converting to a i32 for both vector
15756 // lanes.
15757 SDValue OtherExt(*OtherIt, 0);
15758 if (OtherExt.getValueType() != MVT::i32) {
15759 if (!OtherExt->hasOneUse() ||
15760 OtherExt->user_begin()->getOpcode() != ISD::BITCAST ||
15761 OtherExt->user_begin()->getValueType(0) != MVT::i32)
15762 return SDValue();
15763 OtherExt = SDValue(*OtherExt->user_begin(), 0);
15764 }
15765
15766 // Convert the type to a f64 and extract with a VMOVRRD.
15767 SDValue F64 = DCI.DAG.getNode(
15768 ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
15769 DCI.DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v2f64, Op0),
15770 DCI.DAG.getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15771 SDValue VMOVRRD =
15772 DCI.DAG.getNode(ARMISD::VMOVRRD, dl, {MVT::i32, MVT::i32}, F64);
15773
15774 DCI.CombineTo(OtherExt.getNode(), SDValue(VMOVRRD.getNode(), 1));
15775 return VMOVRRD;
15776}
15777
15780 const ARMSubtarget *ST) {
15781 SDValue Op0 = N->getOperand(0);
15782 EVT VT = N->getValueType(0);
15783 SDLoc dl(N);
15784
15785 // extract (vdup x) -> x
15786 if (Op0->getOpcode() == ARMISD::VDUP) {
15787 SDValue X = Op0->getOperand(0);
15788 if (VT == MVT::f16 && X.getValueType() == MVT::i32)
15789 return DCI.DAG.getNode(ARMISD::VMOVhr, dl, VT, X);
15790 if (VT == MVT::i32 && X.getValueType() == MVT::f16)
15791 return DCI.DAG.getNode(ARMISD::VMOVrh, dl, VT, X);
15792 if (VT == MVT::f32 && X.getValueType() == MVT::i32)
15793 return DCI.DAG.getNode(ISD::BITCAST, dl, VT, X);
15794
15795 while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST)
15796 X = X->getOperand(0);
15797 if (X.getValueType() == VT)
15798 return X;
15799 }
15800
15801 // extract ARM_BUILD_VECTOR -> x
15802 if (Op0->getOpcode() == ARMISD::BUILD_VECTOR &&
15803 isa<ConstantSDNode>(N->getOperand(1)) &&
15804 N->getConstantOperandVal(1) < Op0.getNumOperands()) {
15805 return Op0.getOperand(N->getConstantOperandVal(1));
15806 }
15807
15808 // extract(bitcast(BUILD_VECTOR(VMOVDRR(a, b), ..))) -> a or b
15809 if (Op0.getValueType() == MVT::v4i32 &&
15810 isa<ConstantSDNode>(N->getOperand(1)) &&
15811 Op0.getOpcode() == ISD::BITCAST &&
15813 Op0.getOperand(0).getValueType() == MVT::v2f64) {
15814 SDValue BV = Op0.getOperand(0);
15815 unsigned Offset = N->getConstantOperandVal(1);
15816 SDValue MOV = BV.getOperand(Offset < 2 ? 0 : 1);
15817 if (MOV.getOpcode() == ARMISD::VMOVDRR)
15818 return MOV.getOperand(ST->isLittle() ? Offset % 2 : 1 - Offset % 2);
15819 }
15820
15821 // extract x, n; extract x, n+1 -> VMOVRRD x
15822 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
15823 return R;
15824
15825 // extract (MVETrunc(x)) -> extract x
15826 if (Op0->getOpcode() == ARMISD::MVETRUNC) {
15827 unsigned Idx = N->getConstantOperandVal(1);
15828 unsigned Vec =
15830 unsigned SubIdx =
15832 return DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Op0.getOperand(Vec),
15833 DCI.DAG.getConstant(SubIdx, dl, MVT::i32));
15834 }
15835
15836 // extract(bitcast(BUILD_VECTOR(extract(bitcast(a)), ..))) -> extract(a)
15837 if (ST->isLittle() && Op0.getOpcode() == ISD::BITCAST &&
15839 isa<ConstantSDNode>(N->getOperand(1)) &&
15842 unsigned Lane = N->getConstantOperandVal(1);
15843 EVT ExtVT = Op0.getValueType();
15844 EVT BVVT = Op0.getOperand(0).getValueType();
15845 unsigned BVLane =
15846 (Lane * BVVT.getVectorNumElements()) / ExtVT.getVectorNumElements();
15847 assert(BVLane < Op0.getOperand(0).getNumOperands());
15848 SDValue Ext = Op0.getOperand(0).getOperand(BVLane);
15849 if (Ext.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15850 Ext.getOperand(0).getOpcode() == ISD::BITCAST &&
15852 Ext.getOperand(0).getOperand(0).getValueType() == ExtVT) {
15853 unsigned InnerLane = Ext.getConstantOperandVal(1);
15854 unsigned BVSubLane = Lane - (BVLane * ExtVT.getVectorNumElements()) /
15855 BVVT.getVectorNumElements();
15856 unsigned FinalLane = (InnerLane * ExtVT.getVectorNumElements()) /
15857 BVVT.getVectorNumElements() +
15858 BVSubLane;
15859 return DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT,
15860 Ext.getOperand(0).getOperand(0),
15861 DCI.DAG.getConstant(FinalLane, dl, MVT::i32));
15862 }
15863 }
15864
15865 return SDValue();
15866}
15867
15869 SDValue Op = N->getOperand(0);
15870 EVT VT = N->getValueType(0);
15871
15872 // sext_inreg(VGETLANEu) -> VGETLANEs
15873 if (Op.getOpcode() == ARMISD::VGETLANEu &&
15874 cast<VTSDNode>(N->getOperand(1))->getVT() ==
15875 Op.getOperand(0).getValueType().getScalarType())
15876 return DAG.getNode(ARMISD::VGETLANEs, SDLoc(N), VT, Op.getOperand(0),
15877 Op.getOperand(1));
15878
15879 return SDValue();
15880}
15881
15882static SDValue
15884 SDValue Vec = N->getOperand(0);
15885 SDValue SubVec = N->getOperand(1);
15886 uint64_t IdxVal = N->getConstantOperandVal(2);
15887 EVT VecVT = Vec.getValueType();
15888 EVT SubVT = SubVec.getValueType();
15889
15890 // Only do this for legal fixed vector types.
15891 if (!VecVT.isFixedLengthVector() ||
15892 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VecVT) ||
15894 return SDValue();
15895
15896 // Ignore widening patterns.
15897 if (IdxVal == 0 && Vec.isUndef())
15898 return SDValue();
15899
15900 // Subvector must be half the width and an "aligned" insertion.
15901 unsigned NumSubElts = SubVT.getVectorNumElements();
15902 if ((SubVT.getSizeInBits() * 2) != VecVT.getSizeInBits() ||
15903 (IdxVal != 0 && IdxVal != NumSubElts))
15904 return SDValue();
15905
15906 // Fold insert_subvector -> concat_vectors
15907 // insert_subvector(Vec,Sub,lo) -> concat_vectors(Sub,extract(Vec,hi))
15908 // insert_subvector(Vec,Sub,hi) -> concat_vectors(extract(Vec,lo),Sub)
15909 SDLoc DL(N);
15910 SDValue Lo, Hi;
15911 if (IdxVal == 0) {
15912 Lo = SubVec;
15913 Hi = DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec,
15914 DCI.DAG.getVectorIdxConstant(NumSubElts, DL));
15915 } else {
15916 Lo = DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec,
15917 DCI.DAG.getVectorIdxConstant(0, DL));
15918 Hi = SubVec;
15919 }
15920 return DCI.DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo, Hi);
15921}
15922
15923// shuffle(MVETrunc(x, y)) -> VMOVN(x, y)
15925 SelectionDAG &DAG) {
15926 SDValue Trunc = N->getOperand(0);
15927 EVT VT = Trunc.getValueType();
15928 if (Trunc.getOpcode() != ARMISD::MVETRUNC || !N->getOperand(1).isUndef())
15929 return SDValue();
15930
15931 SDLoc DL(Trunc);
15932 if (isVMOVNTruncMask(N->getMask(), VT, false))
15933 return DAG.getNode(
15934 ARMISD::VMOVN, DL, VT,
15935 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)),
15936 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)),
15937 DAG.getConstant(1, DL, MVT::i32));
15938 else if (isVMOVNTruncMask(N->getMask(), VT, true))
15939 return DAG.getNode(
15940 ARMISD::VMOVN, DL, VT,
15941 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)),
15942 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)),
15943 DAG.getConstant(1, DL, MVT::i32));
15944 return SDValue();
15945}
15946
15947/// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
15948/// ISD::VECTOR_SHUFFLE.
15951 return R;
15952
15953 // The LLVM shufflevector instruction does not require the shuffle mask
15954 // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does
15955 // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the
15956 // operands do not match the mask length, they are extended by concatenating
15957 // them with undef vectors. That is probably the right thing for other
15958 // targets, but for NEON it is better to concatenate two double-register
15959 // size vector operands into a single quad-register size vector. Do that
15960 // transformation here:
15961 // shuffle(concat(v1, undef), concat(v2, undef)) ->
15962 // shuffle(concat(v1, v2), undef)
15963 SDValue Op0 = N->getOperand(0);
15964 SDValue Op1 = N->getOperand(1);
15965 if (Op0.getOpcode() != ISD::CONCAT_VECTORS ||
15966 Op1.getOpcode() != ISD::CONCAT_VECTORS ||
15967 Op0.getNumOperands() != 2 ||
15968 Op1.getNumOperands() != 2)
15969 return SDValue();
15970 SDValue Concat0Op1 = Op0.getOperand(1);
15971 SDValue Concat1Op1 = Op1.getOperand(1);
15972 if (!Concat0Op1.isUndef() || !Concat1Op1.isUndef())
15973 return SDValue();
15974 // Skip the transformation if any of the types are illegal.
15976 EVT VT = N->getValueType(0);
15977 if (!TLI.isTypeLegal(VT) ||
15978 !TLI.isTypeLegal(Concat0Op1.getValueType()) ||
15979 !TLI.isTypeLegal(Concat1Op1.getValueType()))
15980 return SDValue();
15981
15982 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
15983 Op0.getOperand(0), Op1.getOperand(0));
15984 // Translate the shuffle mask.
15985 SmallVector<int, 16> NewMask;
15986 unsigned NumElts = VT.getVectorNumElements();
15987 unsigned HalfElts = NumElts/2;
15989 for (unsigned n = 0; n < NumElts; ++n) {
15990 int MaskElt = SVN->getMaskElt(n);
15991 int NewElt = -1;
15992 if (MaskElt < (int)HalfElts)
15993 NewElt = MaskElt;
15994 else if (MaskElt >= (int)NumElts && MaskElt < (int)(NumElts + HalfElts))
15995 NewElt = HalfElts + MaskElt - NumElts;
15996 NewMask.push_back(NewElt);
15997 }
15998 return DAG.getVectorShuffle(VT, SDLoc(N), NewConcat,
15999 DAG.getUNDEF(VT), NewMask);
16000}
16001
16002/// Load/store instruction that can be merged with a base address
16003/// update
16008 unsigned AddrOpIdx;
16009};
16010
16012 /// Instruction that updates a pointer
16014 /// Pointer increment operand
16016 /// Pointer increment value if it is a constant, or 0 otherwise
16017 unsigned ConstInc;
16018};
16019
16021 // Check that the add is independent of the load/store.
16022 // Otherwise, folding it would create a cycle. Search through Addr
16023 // as well, since the User may not be a direct user of Addr and
16024 // only share a base pointer.
16027 Worklist.push_back(N);
16028 Worklist.push_back(User);
16029 const unsigned MaxSteps = 1024;
16030 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps) ||
16031 SDNode::hasPredecessorHelper(User, Visited, Worklist, MaxSteps))
16032 return false;
16033 return true;
16034}
16035
16037 struct BaseUpdateUser &User,
16038 bool SimpleConstIncOnly,
16040 SelectionDAG &DAG = DCI.DAG;
16041 SDNode *N = Target.N;
16043 SDLoc dl(N);
16044
16045 // Find the new opcode for the updating load/store.
16046 bool isLoadOp = true;
16047 bool isLaneOp = false;
16048 // Workaround for vst1x and vld1x intrinsics which do not have alignment
16049 // as an operand.
16050 bool hasAlignment = true;
16051 unsigned NewOpc = 0;
16052 unsigned NumVecs = 0;
16053 if (Target.isIntrinsic) {
16054 unsigned IntNo = N->getConstantOperandVal(1);
16055 switch (IntNo) {
16056 default:
16057 llvm_unreachable("unexpected intrinsic for Neon base update");
16058 case Intrinsic::arm_neon_vld1:
16059 NewOpc = ARMISD::VLD1_UPD;
16060 NumVecs = 1;
16061 break;
16062 case Intrinsic::arm_neon_vld2:
16063 NewOpc = ARMISD::VLD2_UPD;
16064 NumVecs = 2;
16065 break;
16066 case Intrinsic::arm_neon_vld3:
16067 NewOpc = ARMISD::VLD3_UPD;
16068 NumVecs = 3;
16069 break;
16070 case Intrinsic::arm_neon_vld4:
16071 NewOpc = ARMISD::VLD4_UPD;
16072 NumVecs = 4;
16073 break;
16074 case Intrinsic::arm_neon_vld1x2:
16075 NewOpc = ARMISD::VLD1x2_UPD;
16076 NumVecs = 2;
16077 hasAlignment = false;
16078 break;
16079 case Intrinsic::arm_neon_vld1x3:
16080 NewOpc = ARMISD::VLD1x3_UPD;
16081 NumVecs = 3;
16082 hasAlignment = false;
16083 break;
16084 case Intrinsic::arm_neon_vld1x4:
16085 NewOpc = ARMISD::VLD1x4_UPD;
16086 NumVecs = 4;
16087 hasAlignment = false;
16088 break;
16089 case Intrinsic::arm_neon_vld2dup:
16090 NewOpc = ARMISD::VLD2DUP_UPD;
16091 NumVecs = 2;
16092 break;
16093 case Intrinsic::arm_neon_vld3dup:
16094 NewOpc = ARMISD::VLD3DUP_UPD;
16095 NumVecs = 3;
16096 break;
16097 case Intrinsic::arm_neon_vld4dup:
16098 NewOpc = ARMISD::VLD4DUP_UPD;
16099 NumVecs = 4;
16100 break;
16101 case Intrinsic::arm_neon_vld2lane:
16102 NewOpc = ARMISD::VLD2LN_UPD;
16103 NumVecs = 2;
16104 isLaneOp = true;
16105 break;
16106 case Intrinsic::arm_neon_vld3lane:
16107 NewOpc = ARMISD::VLD3LN_UPD;
16108 NumVecs = 3;
16109 isLaneOp = true;
16110 break;
16111 case Intrinsic::arm_neon_vld4lane:
16112 NewOpc = ARMISD::VLD4LN_UPD;
16113 NumVecs = 4;
16114 isLaneOp = true;
16115 break;
16116 case Intrinsic::arm_neon_vst1:
16117 NewOpc = ARMISD::VST1_UPD;
16118 NumVecs = 1;
16119 isLoadOp = false;
16120 break;
16121 case Intrinsic::arm_neon_vst2:
16122 NewOpc = ARMISD::VST2_UPD;
16123 NumVecs = 2;
16124 isLoadOp = false;
16125 break;
16126 case Intrinsic::arm_neon_vst3:
16127 NewOpc = ARMISD::VST3_UPD;
16128 NumVecs = 3;
16129 isLoadOp = false;
16130 break;
16131 case Intrinsic::arm_neon_vst4:
16132 NewOpc = ARMISD::VST4_UPD;
16133 NumVecs = 4;
16134 isLoadOp = false;
16135 break;
16136 case Intrinsic::arm_neon_vst2lane:
16137 NewOpc = ARMISD::VST2LN_UPD;
16138 NumVecs = 2;
16139 isLoadOp = false;
16140 isLaneOp = true;
16141 break;
16142 case Intrinsic::arm_neon_vst3lane:
16143 NewOpc = ARMISD::VST3LN_UPD;
16144 NumVecs = 3;
16145 isLoadOp = false;
16146 isLaneOp = true;
16147 break;
16148 case Intrinsic::arm_neon_vst4lane:
16149 NewOpc = ARMISD::VST4LN_UPD;
16150 NumVecs = 4;
16151 isLoadOp = false;
16152 isLaneOp = true;
16153 break;
16154 case Intrinsic::arm_neon_vst1x2:
16155 NewOpc = ARMISD::VST1x2_UPD;
16156 NumVecs = 2;
16157 isLoadOp = false;
16158 hasAlignment = false;
16159 break;
16160 case Intrinsic::arm_neon_vst1x3:
16161 NewOpc = ARMISD::VST1x3_UPD;
16162 NumVecs = 3;
16163 isLoadOp = false;
16164 hasAlignment = false;
16165 break;
16166 case Intrinsic::arm_neon_vst1x4:
16167 NewOpc = ARMISD::VST1x4_UPD;
16168 NumVecs = 4;
16169 isLoadOp = false;
16170 hasAlignment = false;
16171 break;
16172 }
16173 } else {
16174 isLaneOp = true;
16175 switch (N->getOpcode()) {
16176 default:
16177 llvm_unreachable("unexpected opcode for Neon base update");
16178 case ARMISD::VLD1DUP:
16179 NewOpc = ARMISD::VLD1DUP_UPD;
16180 NumVecs = 1;
16181 break;
16182 case ARMISD::VLD2DUP:
16183 NewOpc = ARMISD::VLD2DUP_UPD;
16184 NumVecs = 2;
16185 break;
16186 case ARMISD::VLD3DUP:
16187 NewOpc = ARMISD::VLD3DUP_UPD;
16188 NumVecs = 3;
16189 break;
16190 case ARMISD::VLD4DUP:
16191 NewOpc = ARMISD::VLD4DUP_UPD;
16192 NumVecs = 4;
16193 break;
16194 case ISD::LOAD:
16195 NewOpc = ARMISD::VLD1_UPD;
16196 NumVecs = 1;
16197 isLaneOp = false;
16198 break;
16199 case ISD::STORE:
16200 NewOpc = ARMISD::VST1_UPD;
16201 NumVecs = 1;
16202 isLaneOp = false;
16203 isLoadOp = false;
16204 break;
16205 }
16206 }
16207
16208 // Find the size of memory referenced by the load/store.
16209 EVT VecTy;
16210 if (isLoadOp) {
16211 VecTy = N->getValueType(0);
16212 } else if (Target.isIntrinsic) {
16213 VecTy = N->getOperand(Target.AddrOpIdx + 1).getValueType();
16214 } else {
16215 assert(Target.isStore &&
16216 "Node has to be a load, a store, or an intrinsic!");
16217 VecTy = N->getOperand(1).getValueType();
16218 }
16219
16220 bool isVLDDUPOp =
16221 NewOpc == ARMISD::VLD1DUP_UPD || NewOpc == ARMISD::VLD2DUP_UPD ||
16222 NewOpc == ARMISD::VLD3DUP_UPD || NewOpc == ARMISD::VLD4DUP_UPD;
16223
16224 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
16225 if (isLaneOp || isVLDDUPOp)
16226 NumBytes /= VecTy.getVectorNumElements();
16227
16228 if (NumBytes >= 3 * 16 && User.ConstInc != NumBytes) {
16229 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two
16230 // separate instructions that make it harder to use a non-constant update.
16231 return false;
16232 }
16233
16234 if (SimpleConstIncOnly && User.ConstInc != NumBytes)
16235 return false;
16236
16237 if (!isValidBaseUpdate(N, User.N))
16238 return false;
16239
16240 // OK, we found an ADD we can fold into the base update.
16241 // Now, create a _UPD node, taking care of not breaking alignment.
16242
16243 EVT AlignedVecTy = VecTy;
16244 Align Alignment = MemN->getAlign();
16245
16246 // If this is a less-than-standard-aligned load/store, change the type to
16247 // match the standard alignment.
16248 // The alignment is overlooked when selecting _UPD variants; and it's
16249 // easier to introduce bitcasts here than fix that.
16250 // There are 3 ways to get to this base-update combine:
16251 // - intrinsics: they are assumed to be properly aligned (to the standard
16252 // alignment of the memory type), so we don't need to do anything.
16253 // - ARMISD::VLDx nodes: they are only generated from the aforementioned
16254 // intrinsics, so, likewise, there's nothing to do.
16255 // - generic load/store instructions: the alignment is specified as an
16256 // explicit operand, rather than implicitly as the standard alignment
16257 // of the memory type (like the intrinsics). We need to change the
16258 // memory type to match the explicit alignment. That way, we don't
16259 // generate non-standard-aligned ARMISD::VLDx nodes.
16260 if (isa<LSBaseSDNode>(N)) {
16261 if (Alignment.value() < VecTy.getScalarSizeInBits() / 8) {
16262 MVT EltTy = MVT::getIntegerVT(Alignment.value() * 8);
16263 assert(NumVecs == 1 && "Unexpected multi-element generic load/store.");
16264 assert(!isLaneOp && "Unexpected generic load/store lane.");
16265 unsigned NumElts = NumBytes / (EltTy.getSizeInBits() / 8);
16266 AlignedVecTy = MVT::getVectorVT(EltTy, NumElts);
16267 }
16268 // Don't set an explicit alignment on regular load/stores that we want
16269 // to transform to VLD/VST 1_UPD nodes.
16270 // This matches the behavior of regular load/stores, which only get an
16271 // explicit alignment if the MMO alignment is larger than the standard
16272 // alignment of the memory type.
16273 // Intrinsics, however, always get an explicit alignment, set to the
16274 // alignment of the MMO.
16275 Alignment = Align(1);
16276 }
16277
16278 // Create the new updating load/store node.
16279 // First, create an SDVTList for the new updating node's results.
16280 EVT Tys[6];
16281 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16282 unsigned n;
16283 for (n = 0; n < NumResultVecs; ++n)
16284 Tys[n] = AlignedVecTy;
16285 Tys[n++] = MVT::i32;
16286 Tys[n] = MVT::Other;
16287 SDVTList SDTys = DAG.getVTList(ArrayRef(Tys, NumResultVecs + 2));
16288
16289 // Then, gather the new node's operands.
16291 Ops.push_back(N->getOperand(0)); // incoming chain
16292 Ops.push_back(N->getOperand(Target.AddrOpIdx));
16293 Ops.push_back(User.Inc);
16294
16295 if (StoreSDNode *StN = dyn_cast<StoreSDNode>(N)) {
16296 // Try to match the intrinsic's signature
16297 Ops.push_back(StN->getValue());
16298 } else {
16299 // Loads (and of course intrinsics) match the intrinsics' signature,
16300 // so just add all but the alignment operand.
16301 unsigned LastOperand =
16302 hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands();
16303 for (unsigned i = Target.AddrOpIdx + 1; i < LastOperand; ++i)
16304 Ops.push_back(N->getOperand(i));
16305 }
16306
16307 // For all node types, the alignment operand is always the last one.
16308 Ops.push_back(DAG.getConstant(Alignment.value(), dl, MVT::i32));
16309
16310 // If this is a non-standard-aligned STORE, the penultimate operand is the
16311 // stored value. Bitcast it to the aligned type.
16312 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
16313 SDValue &StVal = Ops[Ops.size() - 2];
16314 StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal);
16315 }
16316
16317 EVT LoadVT = isLaneOp ? VecTy.getVectorElementType() : AlignedVecTy;
16318 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, LoadVT,
16319 MemN->getMemOperand());
16320
16321 // Update the uses.
16322 SmallVector<SDValue, 5> NewResults;
16323 for (unsigned i = 0; i < NumResultVecs; ++i)
16324 NewResults.push_back(SDValue(UpdN.getNode(), i));
16325
16326 // If this is an non-standard-aligned LOAD, the first result is the loaded
16327 // value. Bitcast it to the expected result type.
16328 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) {
16329 SDValue &LdVal = NewResults[0];
16330 LdVal = DAG.getNode(ISD::BITCAST, dl, VecTy, LdVal);
16331 }
16332
16333 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
16334 DCI.CombineTo(N, NewResults);
16335 DCI.CombineTo(User.N, SDValue(UpdN.getNode(), NumResultVecs));
16336
16337 return true;
16338}
16339
16340// If (opcode ptr inc) is and ADD-like instruction, return the
16341// increment value. Otherwise return 0.
16342static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr,
16343 SDValue Inc, const SelectionDAG &DAG) {
16345 if (!CInc)
16346 return 0;
16347
16348 switch (Opcode) {
16349 case ARMISD::VLD1_UPD:
16350 case ISD::ADD:
16351 return CInc->getZExtValue();
16352 case ISD::OR: {
16353 if (DAG.haveNoCommonBitsSet(Ptr, Inc)) {
16354 // (OR ptr inc) is the same as (ADD ptr inc)
16355 return CInc->getZExtValue();
16356 }
16357 return 0;
16358 }
16359 default:
16360 return 0;
16361 }
16362}
16363
16365 switch (N->getOpcode()) {
16366 case ISD::ADD:
16367 case ISD::OR: {
16368 if (isa<ConstantSDNode>(N->getOperand(1))) {
16369 *Ptr = N->getOperand(0);
16370 *CInc = N->getOperand(1);
16371 return true;
16372 }
16373 return false;
16374 }
16375 case ARMISD::VLD1_UPD: {
16376 if (isa<ConstantSDNode>(N->getOperand(2))) {
16377 *Ptr = N->getOperand(1);
16378 *CInc = N->getOperand(2);
16379 return true;
16380 }
16381 return false;
16382 }
16383 default:
16384 return false;
16385 }
16386}
16387
16388/// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
16389/// NEON load/store intrinsics, and generic vector load/stores, to merge
16390/// base address updates.
16391/// For generic load/stores, the memory type is assumed to be a vector.
16392/// The caller is assumed to have checked legality.
16395 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID ||
16396 N->getOpcode() == ISD::INTRINSIC_W_CHAIN);
16397 const bool isStore = N->getOpcode() == ISD::STORE;
16398 const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1);
16399 BaseUpdateTarget Target = {N, isIntrinsic, isStore, AddrOpIdx};
16400
16401 // Limit the number of possible base-updates we look at to prevent degenerate
16402 // cases.
16403 unsigned MaxBaseUpdates = ArmMaxBaseUpdatesToCheck;
16404
16405 SDValue Addr = N->getOperand(AddrOpIdx);
16406
16408
16409 // Search for a use of the address operand that is an increment.
16410 for (SDUse &Use : Addr->uses()) {
16411 SDNode *User = Use.getUser();
16412 if (Use.getResNo() != Addr.getResNo() || User->getNumOperands() != 2)
16413 continue;
16414
16415 SDValue Inc = User->getOperand(Use.getOperandNo() == 1 ? 0 : 1);
16416 unsigned ConstInc =
16417 getPointerConstIncrement(User->getOpcode(), Addr, Inc, DCI.DAG);
16418
16419 if (ConstInc || User->getOpcode() == ISD::ADD) {
16420 BaseUpdates.push_back({User, Inc, ConstInc});
16421 if (BaseUpdates.size() >= MaxBaseUpdates)
16422 break;
16423 }
16424 }
16425
16426 // If the address is a constant pointer increment itself, find
16427 // another constant increment that has the same base operand
16428 SDValue Base;
16429 SDValue CInc;
16430 if (findPointerConstIncrement(Addr.getNode(), &Base, &CInc)) {
16431 unsigned Offset =
16432 getPointerConstIncrement(Addr->getOpcode(), Base, CInc, DCI.DAG);
16433 if (Offset) {
16434 for (SDUse &Use : Base->uses()) {
16435
16436 SDNode *User = Use.getUser();
16437 if (Use.getResNo() != Base.getResNo() || User == Addr.getNode() ||
16438 User->getNumOperands() != 2)
16439 continue;
16440
16441 SDValue UserInc = User->getOperand(Use.getOperandNo() == 0 ? 1 : 0);
16442 unsigned UserOffset =
16443 getPointerConstIncrement(User->getOpcode(), Base, UserInc, DCI.DAG);
16444
16445 if (!UserOffset || UserOffset <= Offset)
16446 continue;
16447
16448 unsigned NewConstInc = UserOffset - Offset;
16449 SDValue NewInc = DCI.DAG.getConstant(NewConstInc, SDLoc(N), MVT::i32);
16450 BaseUpdates.push_back({User, NewInc, NewConstInc});
16451 if (BaseUpdates.size() >= MaxBaseUpdates)
16452 break;
16453 }
16454 }
16455 }
16456
16457 // Try to fold the load/store with an update that matches memory
16458 // access size. This should work well for sequential loads.
16459 unsigned NumValidUpd = BaseUpdates.size();
16460 for (unsigned I = 0; I < NumValidUpd; I++) {
16461 BaseUpdateUser &User = BaseUpdates[I];
16462 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/true, DCI))
16463 return SDValue();
16464 }
16465
16466 // Try to fold with other users. Non-constant updates are considered
16467 // first, and constant updates are sorted to not break a sequence of
16468 // strided accesses (if there is any).
16469 llvm::stable_sort(BaseUpdates,
16470 [](const BaseUpdateUser &LHS, const BaseUpdateUser &RHS) {
16471 return LHS.ConstInc < RHS.ConstInc;
16472 });
16473 for (BaseUpdateUser &User : BaseUpdates) {
16474 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/false, DCI))
16475 return SDValue();
16476 }
16477 return SDValue();
16478}
16479
16482 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16483 return SDValue();
16484
16485 return CombineBaseUpdate(N, DCI);
16486}
16487
16490 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16491 return SDValue();
16492
16493 SelectionDAG &DAG = DCI.DAG;
16494 SDValue Addr = N->getOperand(2);
16496 SDLoc dl(N);
16497
16498 // For the stores, where there are multiple intrinsics we only actually want
16499 // to post-inc the last of the them.
16500 unsigned IntNo = N->getConstantOperandVal(1);
16501 if (IntNo == Intrinsic::arm_mve_vst2q && N->getConstantOperandVal(5) != 1)
16502 return SDValue();
16503 if (IntNo == Intrinsic::arm_mve_vst4q && N->getConstantOperandVal(7) != 3)
16504 return SDValue();
16505
16506 // Search for a use of the address operand that is an increment.
16507 for (SDUse &Use : Addr->uses()) {
16508 SDNode *User = Use.getUser();
16509 if (User->getOpcode() != ISD::ADD || Use.getResNo() != Addr.getResNo())
16510 continue;
16511
16512 // Check that the add is independent of the load/store. Otherwise, folding
16513 // it would create a cycle. We can avoid searching through Addr as it's a
16514 // predecessor to both.
16517 Visited.insert(Addr.getNode());
16518 Worklist.push_back(N);
16519 Worklist.push_back(User);
16520 const unsigned MaxSteps = 1024;
16521 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps) ||
16522 SDNode::hasPredecessorHelper(User, Visited, Worklist, MaxSteps))
16523 continue;
16524
16525 // Find the new opcode for the updating load/store.
16526 bool isLoadOp = true;
16527 unsigned NewOpc = 0;
16528 unsigned NumVecs = 0;
16529 switch (IntNo) {
16530 default:
16531 llvm_unreachable("unexpected intrinsic for MVE VLDn combine");
16532 case Intrinsic::arm_mve_vld2q:
16533 NewOpc = ARMISD::VLD2_UPD;
16534 NumVecs = 2;
16535 break;
16536 case Intrinsic::arm_mve_vld4q:
16537 NewOpc = ARMISD::VLD4_UPD;
16538 NumVecs = 4;
16539 break;
16540 case Intrinsic::arm_mve_vst2q:
16541 NewOpc = ARMISD::VST2_UPD;
16542 NumVecs = 2;
16543 isLoadOp = false;
16544 break;
16545 case Intrinsic::arm_mve_vst4q:
16546 NewOpc = ARMISD::VST4_UPD;
16547 NumVecs = 4;
16548 isLoadOp = false;
16549 break;
16550 }
16551
16552 // Find the size of memory referenced by the load/store.
16553 EVT VecTy;
16554 if (isLoadOp) {
16555 VecTy = N->getValueType(0);
16556 } else {
16557 VecTy = N->getOperand(3).getValueType();
16558 }
16559
16560 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
16561
16562 // If the increment is a constant, it must match the memory ref size.
16563 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
16565 if (!CInc || CInc->getZExtValue() != NumBytes)
16566 continue;
16567
16568 // Create the new updating load/store node.
16569 // First, create an SDVTList for the new updating node's results.
16570 EVT Tys[6];
16571 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16572 unsigned n;
16573 for (n = 0; n < NumResultVecs; ++n)
16574 Tys[n] = VecTy;
16575 Tys[n++] = MVT::i32;
16576 Tys[n] = MVT::Other;
16577 SDVTList SDTys = DAG.getVTList(ArrayRef(Tys, NumResultVecs + 2));
16578
16579 // Then, gather the new node's operands.
16581 Ops.push_back(N->getOperand(0)); // incoming chain
16582 Ops.push_back(N->getOperand(2)); // ptr
16583 Ops.push_back(Inc);
16584
16585 for (unsigned i = 3; i < N->getNumOperands(); ++i)
16586 Ops.push_back(N->getOperand(i));
16587
16588 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, VecTy,
16589 MemN->getMemOperand());
16590
16591 // Update the uses.
16592 SmallVector<SDValue, 5> NewResults;
16593 for (unsigned i = 0; i < NumResultVecs; ++i)
16594 NewResults.push_back(SDValue(UpdN.getNode(), i));
16595
16596 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
16597 DCI.CombineTo(N, NewResults);
16598 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
16599
16600 break;
16601 }
16602
16603 return SDValue();
16604}
16605
16606/// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
16607/// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
16608/// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
16609/// return true.
16611 SelectionDAG &DAG = DCI.DAG;
16612 EVT VT = N->getValueType(0);
16613 // vldN-dup instructions only support 64-bit vectors for N > 1.
16614 if (!VT.is64BitVector())
16615 return false;
16616
16617 // Check if the VDUPLANE operand is a vldN-dup intrinsic.
16618 SDNode *VLD = N->getOperand(0).getNode();
16619 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN)
16620 return false;
16621 unsigned NumVecs = 0;
16622 unsigned NewOpc = 0;
16623 unsigned IntNo = VLD->getConstantOperandVal(1);
16624 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16625 NumVecs = 2;
16626 NewOpc = ARMISD::VLD2DUP;
16627 } else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16628 NumVecs = 3;
16629 NewOpc = ARMISD::VLD3DUP;
16630 } else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16631 NumVecs = 4;
16632 NewOpc = ARMISD::VLD4DUP;
16633 } else {
16634 return false;
16635 }
16636
16637 // First check that all the vldN-lane uses are VDUPLANEs and that the lane
16638 // numbers match the load.
16639 unsigned VLDLaneNo = VLD->getConstantOperandVal(NumVecs + 3);
16640 for (SDUse &Use : VLD->uses()) {
16641 // Ignore uses of the chain result.
16642 if (Use.getResNo() == NumVecs)
16643 continue;
16644 SDNode *User = Use.getUser();
16645 if (User->getOpcode() != ARMISD::VDUPLANE ||
16646 VLDLaneNo != User->getConstantOperandVal(1))
16647 return false;
16648 }
16649
16650 // Create the vldN-dup node.
16651 EVT Tys[5];
16652 unsigned n;
16653 for (n = 0; n < NumVecs; ++n)
16654 Tys[n] = VT;
16655 Tys[n] = MVT::Other;
16656 SDVTList SDTys = DAG.getVTList(ArrayRef(Tys, NumVecs + 1));
16657 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) };
16659 SDValue VLDDup = DAG.getMemIntrinsicNode(NewOpc, SDLoc(VLD), SDTys,
16660 Ops, VLDMemInt->getMemoryVT(),
16661 VLDMemInt->getMemOperand());
16662
16663 // Update the uses.
16664 for (SDUse &Use : VLD->uses()) {
16665 unsigned ResNo = Use.getResNo();
16666 // Ignore uses of the chain result.
16667 if (ResNo == NumVecs)
16668 continue;
16669 DCI.CombineTo(Use.getUser(), SDValue(VLDDup.getNode(), ResNo));
16670 }
16671
16672 // Now the vldN-lane intrinsic is dead except for its chain result.
16673 // Update uses of the chain.
16674 std::vector<SDValue> VLDDupResults;
16675 for (unsigned n = 0; n < NumVecs; ++n)
16676 VLDDupResults.push_back(SDValue(VLDDup.getNode(), n));
16677 VLDDupResults.push_back(SDValue(VLDDup.getNode(), NumVecs));
16678 DCI.CombineTo(VLD, VLDDupResults);
16679
16680 return true;
16681}
16682
16683/// PerformVDUPLANECombine - Target-specific dag combine xforms for
16684/// ARMISD::VDUPLANE.
16687 const ARMSubtarget *Subtarget) {
16688 SDValue Op = N->getOperand(0);
16689 EVT VT = N->getValueType(0);
16690
16691 // On MVE, we just convert the VDUPLANE to a VDUP with an extract.
16692 if (Subtarget->hasMVEIntegerOps()) {
16693 EVT ExtractVT = VT.getVectorElementType();
16694 // We need to ensure we are creating a legal type.
16695 if (!DCI.DAG.getTargetLoweringInfo().isTypeLegal(ExtractVT))
16696 ExtractVT = MVT::i32;
16697 SDValue Extract = DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), ExtractVT,
16698 N->getOperand(0), N->getOperand(1));
16699 return DCI.DAG.getNode(ARMISD::VDUP, SDLoc(N), VT, Extract);
16700 }
16701
16702 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses
16703 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation.
16704 if (CombineVLDDUP(N, DCI))
16705 return SDValue(N, 0);
16706
16707 // If the source is already a VMOVIMM or VMVNIMM splat, the VDUPLANE is
16708 // redundant. Ignore bit_converts for now; element sizes are checked below.
16709 while (Op.getOpcode() == ISD::BITCAST)
16710 Op = Op.getOperand(0);
16711 if (Op.getOpcode() != ARMISD::VMOVIMM && Op.getOpcode() != ARMISD::VMVNIMM)
16712 return SDValue();
16713
16714 // Make sure the VMOV element size is not bigger than the VDUPLANE elements.
16715 unsigned EltSize = Op.getScalarValueSizeInBits();
16716 // The canonical VMOV for a zero vector uses a 32-bit element size.
16717 unsigned Imm = Op.getConstantOperandVal(0);
16718 unsigned EltBits;
16719 if (ARM_AM::decodeVMOVModImm(Imm, EltBits) == 0)
16720 EltSize = 8;
16721 if (EltSize > VT.getScalarSizeInBits())
16722 return SDValue();
16723
16724 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
16725}
16726
16727/// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
16729 const ARMSubtarget *Subtarget) {
16730 SDValue Op = N->getOperand(0);
16731 SDLoc dl(N);
16732
16733 if (Subtarget->hasMVEIntegerOps()) {
16734 // Convert VDUP f32 -> VDUP BITCAST i32 under MVE, as we know the value will
16735 // need to come from a GPR.
16736 if (Op.getValueType() == MVT::f32)
16737 return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0),
16738 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op));
16739 else if (Op.getValueType() == MVT::f16)
16740 return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0),
16741 DAG.getNode(ARMISD::VMOVrh, dl, MVT::i32, Op));
16742 }
16743
16744 if (!Subtarget->hasNEON())
16745 return SDValue();
16746
16747 // Match VDUP(LOAD) -> VLD1DUP.
16748 // We match this pattern here rather than waiting for isel because the
16749 // transform is only legal for unindexed loads.
16750 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode());
16751 if (LD && Op.hasOneUse() && LD->isUnindexed() &&
16752 LD->getMemoryVT() == N->getValueType(0).getVectorElementType()) {
16753 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16754 DAG.getConstant(LD->getAlign().value(), SDLoc(N), MVT::i32)};
16755 SDVTList SDTys = DAG.getVTList(N->getValueType(0), MVT::Other);
16756 SDValue VLDDup =
16758 LD->getMemoryVT(), LD->getMemOperand());
16759 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), VLDDup.getValue(1));
16760 return VLDDup;
16761 }
16762
16763 return SDValue();
16764}
16765
16768 const ARMSubtarget *Subtarget) {
16769 EVT VT = N->getValueType(0);
16770 // Let the deinterleave combine form a structured load first.
16772 for (SDUse &Use : N->uses())
16773 if (Use.getUser()->getOpcode() == ISD::VECTOR_DEINTERLEAVE)
16774 return SDValue();
16775 // If this is a legal vector load, try to combine it into a VLD1_UPD.
16776 if (Subtarget->hasNEON() && ISD::isNormalLoad(N) && VT.isVector() &&
16778 return CombineBaseUpdate(N, DCI);
16779
16780 return SDValue();
16781}
16782
16783// Optimize trunc store (of multiple scalars) to shuffle and store. First,
16784// pack all of the elements in one place. Next, store to memory in fewer
16785// chunks.
16787 SelectionDAG &DAG) {
16788 SDValue StVal = St->getValue();
16789 EVT VT = StVal.getValueType();
16790 if (!St->isTruncatingStore() || !VT.isVector())
16791 return SDValue();
16793 EVT StVT = St->getMemoryVT();
16794 unsigned NumElems = VT.getVectorNumElements();
16795 assert(StVT != VT && "Cannot truncate to the same type");
16796 unsigned FromEltSz = VT.getScalarSizeInBits();
16797 unsigned ToEltSz = StVT.getScalarSizeInBits();
16798
16799 // From, To sizes and ElemCount must be pow of two
16800 if (!isPowerOf2_32(NumElems * FromEltSz * ToEltSz))
16801 return SDValue();
16802
16803 // We are going to use the original vector elt for storing.
16804 // Accumulated smaller vector elements must be a multiple of the store size.
16805 if (0 != (NumElems * FromEltSz) % ToEltSz)
16806 return SDValue();
16807
16808 unsigned SizeRatio = FromEltSz / ToEltSz;
16809 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits());
16810
16811 // Create a type on which we perform the shuffle.
16812 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), StVT.getScalarType(),
16813 NumElems * SizeRatio);
16814 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
16815
16816 SDLoc DL(St);
16817 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
16818 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
16819 for (unsigned i = 0; i < NumElems; ++i)
16820 ShuffleVec[i] = DAG.getDataLayout().isBigEndian() ? (i + 1) * SizeRatio - 1
16821 : i * SizeRatio;
16822
16823 // Can't shuffle using an illegal type.
16824 if (!TLI.isTypeLegal(WideVecVT))
16825 return SDValue();
16826
16827 SDValue Shuff = DAG.getVectorShuffle(
16828 WideVecVT, DL, WideVec, DAG.getUNDEF(WideVec.getValueType()), ShuffleVec);
16829 // At this point all of the data is stored at the bottom of the
16830 // register. We now need to save it to mem.
16831
16832 // Find the largest store unit
16833 MVT StoreType = MVT::i8;
16834 for (MVT Tp : MVT::integer_valuetypes()) {
16835 if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16836 StoreType = Tp;
16837 }
16838 // Didn't find a legal store type.
16839 if (!TLI.isTypeLegal(StoreType))
16840 return SDValue();
16841
16842 // Bitcast the original vector into a vector of store-size units
16843 EVT StoreVecVT =
16844 EVT::getVectorVT(*DAG.getContext(), StoreType,
16845 VT.getSizeInBits() / EVT(StoreType).getSizeInBits());
16846 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
16847 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
16849 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
16850 TLI.getPointerTy(DAG.getDataLayout()));
16851 SDValue BasePtr = St->getBasePtr();
16852
16853 // Perform one or more big stores into memory.
16854 unsigned E = (ToEltSz * NumElems) / StoreType.getSizeInBits();
16855 for (unsigned I = 0; I < E; I++) {
16856 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreType,
16857 ShuffWide, DAG.getIntPtrConstant(I, DL));
16858 SDValue Ch =
16859 DAG.getStore(St->getChain(), DL, SubVec, BasePtr, St->getPointerInfo(),
16860 St->getAlign(), St->getMemOperand()->getFlags());
16861 BasePtr =
16862 DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr, Increment);
16863 Chains.push_back(Ch);
16864 }
16865 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
16866}
16867
16868// Try taking a single vector store from an fpround (which would otherwise turn
16869// into an expensive buildvector) and splitting it into a series of narrowing
16870// stores.
16872 SelectionDAG &DAG) {
16873 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16874 return SDValue();
16875 SDValue Trunc = St->getValue();
16876 if (Trunc->getOpcode() != ISD::FP_ROUND)
16877 return SDValue();
16878 EVT FromVT = Trunc->getOperand(0).getValueType();
16879 EVT ToVT = Trunc.getValueType();
16880 if (!ToVT.isVector())
16881 return SDValue();
16883 EVT ToEltVT = ToVT.getVectorElementType();
16884 EVT FromEltVT = FromVT.getVectorElementType();
16885
16886 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16887 return SDValue();
16888
16889 unsigned NumElements = 4;
16890 if (FromVT.getVectorNumElements() % NumElements != 0)
16891 return SDValue();
16892
16893 // Test if the Trunc will be convertible to a VMOVN with a shuffle, and if so
16894 // use the VMOVN over splitting the store. We are looking for patterns of:
16895 // !rev: 0 N 1 N+1 2 N+2 ...
16896 // rev: N 0 N+1 1 N+2 2 ...
16897 // The shuffle may either be a single source (in which case N = NumElts/2) or
16898 // two inputs extended with concat to the same size (in which case N =
16899 // NumElts).
16900 auto isVMOVNShuffle = [&](ShuffleVectorSDNode *SVN, bool Rev) {
16901 ArrayRef<int> M = SVN->getMask();
16902 unsigned NumElts = ToVT.getVectorNumElements();
16903 if (SVN->getOperand(1).isUndef())
16904 NumElts /= 2;
16905
16906 unsigned Off0 = Rev ? NumElts : 0;
16907 unsigned Off1 = Rev ? 0 : NumElts;
16908
16909 for (unsigned I = 0; I < NumElts; I += 2) {
16910 if (M[I] >= 0 && M[I] != (int)(Off0 + I / 2))
16911 return false;
16912 if (M[I + 1] >= 0 && M[I + 1] != (int)(Off1 + I / 2))
16913 return false;
16914 }
16915
16916 return true;
16917 };
16918
16919 if (auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.getOperand(0)))
16920 if (isVMOVNShuffle(Shuffle, false) || isVMOVNShuffle(Shuffle, true))
16921 return SDValue();
16922
16923 LLVMContext &C = *DAG.getContext();
16924 SDLoc DL(St);
16925 // Details about the old store
16926 SDValue Ch = St->getChain();
16927 SDValue BasePtr = St->getBasePtr();
16928 Align Alignment = St->getBaseAlign();
16929 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16930 AAMDNodes AAInfo = St->getAAInfo();
16931
16932 // We split the store into slices of NumElements. fp16 trunc stores are vcvt
16933 // and then stored as truncating integer stores.
16934 EVT NewFromVT = EVT::getVectorVT(C, FromEltVT, NumElements);
16935 EVT NewToVT = EVT::getVectorVT(
16936 C, EVT::getIntegerVT(C, ToEltVT.getSizeInBits()), NumElements);
16937
16939 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
16940 unsigned NewOffset = i * NumElements * ToEltVT.getSizeInBits() / 8;
16941 SDValue NewPtr =
16942 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(NewOffset));
16943
16944 SDValue Extract =
16945 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewFromVT, Trunc.getOperand(0),
16946 DAG.getConstant(i * NumElements, DL, MVT::i32));
16947
16948 SDValue FPTrunc =
16949 DAG.getNode(ARMISD::VCVTN, DL, MVT::v8f16, DAG.getUNDEF(MVT::v8f16),
16950 Extract, DAG.getConstant(0, DL, MVT::i32));
16951 Extract = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v4i32, FPTrunc);
16952
16954 Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
16955 NewToVT, Alignment, MMOFlags, AAInfo);
16956 Stores.push_back(Store);
16957 }
16958 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
16959}
16960
16961// Try taking a single vector store from an MVETRUNC (which would otherwise turn
16962// into an expensive buildvector) and splitting it into a series of narrowing
16963// stores.
16965 SelectionDAG &DAG) {
16966 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16967 return SDValue();
16968 SDValue Trunc = St->getValue();
16969 if (Trunc->getOpcode() != ARMISD::MVETRUNC)
16970 return SDValue();
16971 EVT FromVT = Trunc->getOperand(0).getValueType();
16972 EVT ToVT = Trunc.getValueType();
16973
16974 LLVMContext &C = *DAG.getContext();
16975 SDLoc DL(St);
16976 // Details about the old store
16977 SDValue Ch = St->getChain();
16978 SDValue BasePtr = St->getBasePtr();
16979 Align Alignment = St->getBaseAlign();
16980 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16981 AAMDNodes AAInfo = St->getAAInfo();
16982
16983 EVT NewToVT = EVT::getVectorVT(C, ToVT.getVectorElementType(),
16984 FromVT.getVectorNumElements());
16985
16987 for (unsigned i = 0; i < Trunc.getNumOperands(); i++) {
16988 unsigned NewOffset =
16989 i * FromVT.getVectorNumElements() * ToVT.getScalarSizeInBits() / 8;
16990 SDValue NewPtr =
16991 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(NewOffset));
16992
16993 SDValue Extract = Trunc.getOperand(i);
16995 Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
16996 NewToVT, Alignment, MMOFlags, AAInfo);
16997 Stores.push_back(Store);
16998 }
16999 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
17000}
17001
17002// Given a floating point store from an extracted vector, with an integer
17003// VGETLANE that already exists, store the existing VGETLANEu directly. This can
17004// help reduce fp register pressure, doesn't require the fp extract and allows
17005// use of more integer post-inc stores not available with vstr.
17007 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
17008 return SDValue();
17009 SDValue Extract = St->getValue();
17010 EVT VT = Extract.getValueType();
17011 // For now only uses f16. This may be useful for f32 too, but that will
17012 // be bitcast(extract), not the VGETLANEu we currently check here.
17013 if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
17014 return SDValue();
17015
17016 SDNode *GetLane =
17017 DAG.getNodeIfExists(ARMISD::VGETLANEu, DAG.getVTList(MVT::i32),
17018 {Extract.getOperand(0), Extract.getOperand(1)});
17019 if (!GetLane)
17020 return SDValue();
17021
17022 LLVMContext &C = *DAG.getContext();
17023 SDLoc DL(St);
17024 // Create a new integer store to replace the existing floating point version.
17025 SDValue Ch = St->getChain();
17026 SDValue BasePtr = St->getBasePtr();
17027 Align Alignment = St->getBaseAlign();
17028 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
17029 AAMDNodes AAInfo = St->getAAInfo();
17030 EVT NewToVT = EVT::getIntegerVT(C, VT.getSizeInBits());
17031 SDValue Store = DAG.getTruncStore(Ch, DL, SDValue(GetLane, 0), BasePtr,
17032 St->getPointerInfo(), NewToVT, Alignment,
17033 MMOFlags, AAInfo);
17034
17035 return Store;
17036}
17037
17038// Combine stores of all VECTOR_INTERLEAVE results into a structured store
17039// (vstN) when they access consecutive memory. Running this after type
17040// legalization let wider vector types to be split into legal types, so each
17041// split can form a structured store.
17044 const ARMSubtarget *Subtarget) {
17046 return SDValue();
17047
17048 if (!ISD::isNormalStore(ST) || !ST->isSimple() || !ST->getOffset().isUndef())
17049 return SDValue();
17050
17051 SDValue StoredValue = ST->getValue();
17052 SDNode *Interleave = StoredValue.getNode();
17053 if (Interleave->getOpcode() != ISD::VECTOR_INTERLEAVE)
17054 return SDValue();
17055
17056 unsigned NumParts = Interleave->getNumOperands();
17058 return SDValue();
17059
17060 if (NumParts == 3 && Subtarget->hasMVEIntegerOps())
17061 return SDValue();
17062
17063 EVT SubVecTy = Interleave->getValueType(0);
17064 if (Subtarget->hasNEON() && !SubVecTy.is64BitVector() &&
17065 !SubVecTy.is128BitVector())
17066 return SDValue();
17067 if (Subtarget->hasMVEIntegerOps() && !SubVecTy.is128BitVector())
17068 return SDValue();
17069 unsigned EltBits = SubVecTy.getScalarSizeInBits();
17070 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
17071 return SDValue();
17072
17073 SmallVector<StoreSDNode *, 4> Stores(NumParts, nullptr);
17074 for (SDUse &Use : Interleave->uses()) {
17075 unsigned ResNo = Use.getResNo();
17077 if (Stores[ResNo] || !Store)
17078 return SDValue();
17079 Stores[ResNo] = Store;
17080 }
17081
17082 if (llvm::is_contained(Stores, nullptr))
17083 return SDValue();
17084
17085 StoreSDNode *BaseStore = Stores[0];
17086 if (Subtarget->hasMVEIntegerOps() && BaseStore->getAlign() < EltBits / 8)
17087 return SDValue();
17088 unsigned Bytes = SubVecTy.getStoreSize().getFixedValue();
17089 for (auto [Idx, Store] : enumerate(Stores)) {
17090 if (!DAG.areNonVolatileConsecutiveStores(Store, BaseStore, Bytes, Idx))
17091 return SDValue();
17092 }
17093
17094 static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::arm_neon_vst2,
17095 Intrinsic::arm_neon_vst3,
17096 Intrinsic::arm_neon_vst4};
17097 SDLoc DL(Interleave);
17099
17100 Ops.push_back(BaseStore->getChain());
17101 Intrinsic::ID IID = 0;
17102 if (Subtarget->hasNEON())
17103 IID = NEONStores[NumParts - 2];
17104 else
17105 IID = NumParts == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
17106
17107 Ops.push_back(DAG.getTargetConstant(IID, DL, MVT::i64));
17108
17109 Ops.push_back(BaseStore->getBasePtr());
17110 Ops.append(Interleave->op_begin(), Interleave->op_end());
17111 Ops.push_back(DAG.getConstant(BaseStore->getAlign().value(), DL, MVT::i32));
17112
17113 EVT MemVT =
17114 EVT::getVectorVT(*DAG.getContext(), SubVecTy.getVectorElementType(),
17115 SubVecTy.getVectorElementCount() * NumParts);
17117 MachineMemOperand *MMO =
17118 MF.getMachineMemOperand(BaseStore->getMemOperand(), 0, NumParts * Bytes);
17119 SDValue NewStore;
17120 if (Subtarget->hasNEON()) {
17121 NewStore = DAG.getMemIntrinsicNode(
17122 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
17123 } else {
17124 // MVE
17125 for (unsigned Stage = 0; Stage < NumParts; Stage++) {
17126 Ops.back() = DAG.getConstant(Stage, DL, MVT::i32);
17127 NewStore = DAG.getMemIntrinsicNode(
17128 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
17129 Ops.front() = NewStore;
17130 }
17131 }
17132 for (StoreSDNode *Store : Stores) {
17133 DAG.makeEquivalentMemoryOrdering(SDValue(Store, 0), NewStore);
17134 if (Store != ST)
17135 DCI.CombineTo(Store, NewStore);
17136 }
17137 return NewStore;
17138}
17139
17140static bool
17142 if (N->getOpcode() != ISD::CONCAT_VECTORS)
17143 return false;
17144
17145 unsigned NumParts = N->getNumOperands();
17146
17147 // We should be concatenating each sequential result from a
17148 // VECTOR_INTERLEAVE.
17149 SDNode *InterleaveOp = N->getOperand(0).getNode();
17150 if (InterleaveOp->getOpcode() != ISD::VECTOR_INTERLEAVE ||
17151 InterleaveOp->getNumOperands() != NumParts)
17152 return false;
17153
17154 for (unsigned I = 0; I < NumParts; I++)
17155 if (N->getOperand(I) != SDValue(InterleaveOp, I))
17156 return false;
17157
17158 Ops.append(InterleaveOp->op_begin(), InterleaveOp->op_end());
17159 return true;
17160}
17161
17164 const ARMSubtarget *Subtarget) {
17165 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
17166 return SDValue();
17167 if (SDValue Res =
17169 return Res;
17170
17171 if (!DCI.isBeforeLegalize())
17172 return SDValue();
17173
17174 if (!ISD::isNormalStore(ST) || !ST->isSimple() || !ST->getOffset().isUndef())
17175 return SDValue();
17176
17177 SDValue WideValue = ST->getValue();
17178 SmallVector<SDValue, 4> ValueInterleaveOps;
17180 ValueInterleaveOps))
17181 return SDValue();
17182
17183 if (!WideValue.hasOneUse())
17184 return SDValue();
17185
17186 unsigned NumParts = ValueInterleaveOps.size();
17187 if (NumParts != 2 && NumParts != 3 && NumParts != 4)
17188 return SDValue();
17189
17190 if (NumParts == 3 && Subtarget->hasMVEIntegerOps())
17191 return SDValue();
17192
17193 EVT SubVecTy = ValueInterleaveOps[0].getValueType();
17194 if (Subtarget->hasNEON() && !SubVecTy.is64BitVector() &&
17195 !SubVecTy.is128BitVector())
17196 return SDValue();
17197 if (Subtarget->hasMVEIntegerOps() && !SubVecTy.is128BitVector())
17198 return SDValue();
17199 unsigned EltBits = SubVecTy.getScalarSizeInBits();
17200 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
17201 return SDValue();
17202 if (Subtarget->hasMVEIntegerOps() && ST->getAlign() < EltBits / 8)
17203 return SDValue();
17204 static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::arm_neon_vst2,
17205 Intrinsic::arm_neon_vst3,
17206 Intrinsic::arm_neon_vst4};
17207 SDLoc DL(ST);
17209 Ops.push_back(ST->getChain());
17210 Intrinsic::ID IID = 0;
17211 if (Subtarget->hasNEON())
17212 IID = NEONStores[NumParts - 2];
17213 else
17214 IID = NumParts == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
17215
17216 Ops.push_back(DAG.getTargetConstant(IID, DL, MVT::i64));
17217
17218 Ops.push_back(ST->getBasePtr());
17219 Ops.append(ValueInterleaveOps);
17220
17221 Ops.push_back(DAG.getConstant(ST->getAlign().value(), DL, MVT::i32));
17222 if (Subtarget->hasNEON())
17224 DAG.getVTList(MVT::Other), Ops,
17225 ST->getMemoryVT(), ST->getMemOperand());
17226 // MVE
17227 SDValue NewStore;
17228 for (unsigned Stage = 0; Stage < NumParts; Stage++) {
17229 Ops.back() = DAG.getConstant(Stage, DL, MVT::i32);
17231 DAG.getVTList(MVT::Other), Ops,
17232 ST->getMemoryVT(), ST->getMemOperand());
17233 Ops.front() = NewStore;
17234 }
17235 return NewStore;
17236}
17237
17238/// PerformSTORECombine - Target-specific dag combine xforms for
17239/// ISD::STORE.
17242 const ARMSubtarget *Subtarget) {
17244 if (St->isVolatile())
17245 return SDValue();
17246 SDValue StVal = St->getValue();
17247 EVT VT = StVal.getValueType();
17248 if (SDValue Store =
17250 return Store;
17251
17252 if (Subtarget->hasNEON())
17254 return Store;
17255
17256 if (Subtarget->hasMVEFloatOps())
17257 if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DCI.DAG))
17258 return NewToken;
17259
17260 if (Subtarget->hasMVEIntegerOps()) {
17261 if (SDValue NewChain = PerformExtractFpToIntStores(St, DCI.DAG))
17262 return NewChain;
17263 if (SDValue NewToken =
17265 return NewToken;
17266 }
17267
17268 if (!ISD::isNormalStore(St))
17269 return SDValue();
17270
17271 // Split a store of a VMOVDRR into two integer stores to avoid mixing NEON and
17272 // ARM stores of arguments in the same cache line.
17273 if (StVal.getOpcode() == ARMISD::VMOVDRR && StVal->hasOneUse()) {
17274 SelectionDAG &DAG = DCI.DAG;
17275 bool isBigEndian = DAG.getDataLayout().isBigEndian();
17276 SDLoc DL(St);
17277 SDValue BasePtr = St->getBasePtr();
17278 SDValue NewST1 =
17279 DAG.getStore(St->getChain(), DL, StVal.getOperand(isBigEndian ? 1 : 0),
17280 BasePtr, St->getPointerInfo(), St->getBaseAlign(),
17281 St->getMemOperand()->getFlags());
17282
17283 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
17284 DAG.getConstant(4, DL, MVT::i32));
17285 return DAG.getStore(NewST1.getValue(0), DL,
17286 StVal.getOperand(isBigEndian ? 0 : 1), OffsetPtr,
17287 St->getPointerInfo().getWithOffset(4),
17288 St->getBaseAlign(), St->getMemOperand()->getFlags());
17289 }
17290
17291 if (StVal.getValueType() == MVT::i64 &&
17293 // Bitcast an i64 store extracted from a vector to f64.
17294 // Otherwise, the i64 value will be legalized to a pair of i32 values.
17295 SelectionDAG &DAG = DCI.DAG;
17296 SDLoc dl(StVal);
17297 SDValue IntVec = StVal.getOperand(0);
17298 EVT FloatVT =
17299 EVT::getVectorVT(*DAG.getContext(), MVT::f64,
17301 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec);
17302 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Vec,
17303 StVal.getOperand(1));
17304 dl = SDLoc(N);
17305 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt);
17306 // Make the DAGCombiner fold the bitcasts.
17307 DCI.AddToWorklist(Vec.getNode());
17308 DCI.AddToWorklist(ExtElt.getNode());
17309 DCI.AddToWorklist(V.getNode());
17310 return DAG.getStore(St->getChain(), dl, V, St->getBasePtr(),
17311 St->getPointerInfo(), St->getAlign(),
17312 St->getMemOperand()->getFlags(), St->getAAInfo());
17313 }
17314
17315 // If this is a legal vector store, try to combine it into a VST1_UPD.
17316 if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() &&
17318 return CombineBaseUpdate(N, DCI);
17319
17320 return SDValue();
17321}
17322
17323/// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
17324/// can replace combinations of VMUL and VCVT (floating-point to integer)
17325/// when the VMUL has a constant operand that is a power of 2.
17326///
17327/// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
17328/// vmul.f32 d16, d17, d16
17329/// vcvt.s32.f32 d16, d16
17330/// becomes:
17331/// vcvt.s32.f32 d16, d16, #3
17333 const ARMSubtarget *Subtarget) {
17334 if (!Subtarget->hasNEON())
17335 return SDValue();
17336
17337 SDValue Op = N->getOperand(0);
17338 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
17339 Op.getOpcode() != ISD::FMUL)
17340 return SDValue();
17341
17342 SDValue ConstVec = Op->getOperand(1);
17343 if (!isa<BuildVectorSDNode>(ConstVec))
17344 return SDValue();
17345
17346 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
17347 uint32_t FloatBits = FloatTy.getSizeInBits();
17348 MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
17349 uint32_t IntBits = IntTy.getSizeInBits();
17350 unsigned NumLanes = Op.getValueType().getVectorNumElements();
17351 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17352 // These instructions only exist converting from f32 to i32. We can handle
17353 // smaller integers by generating an extra truncate, but larger ones would
17354 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
17355 // these instructions only support v2i32/v4i32 types.
17356 return SDValue();
17357 }
17358
17359 BitVector UndefElements;
17361 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33);
17362 if (C == -1 || C == 0 || C > 32)
17363 return SDValue();
17364
17365 SDLoc dl(N);
17366 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT;
17367 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
17368 Intrinsic::arm_neon_vcvtfp2fxu;
17369 SDValue FixConv = DAG.getNode(
17370 ISD::INTRINSIC_WO_CHAIN, dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
17371 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), Op->getOperand(0),
17372 DAG.getConstant(C, dl, MVT::i32));
17373
17374 if (IntBits < FloatBits)
17375 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv);
17376
17377 return FixConv;
17378}
17379
17381 const ARMSubtarget *Subtarget) {
17382 if (!Subtarget->hasMVEFloatOps())
17383 return SDValue();
17384
17385 // Turn (fadd x, (vselect c, y, -0.0)) into (vselect c, (fadd x, y), x)
17386 // The second form can be more easily turned into a predicated vadd, and
17387 // possibly combined into a fma to become a predicated vfma.
17388 SDValue Op0 = N->getOperand(0);
17389 SDValue Op1 = N->getOperand(1);
17390 EVT VT = N->getValueType(0);
17391 SDLoc DL(N);
17392
17393 // The identity element for a fadd is -0.0 or +0.0 when the nsz flag is set,
17394 // which these VMOV's represent.
17395 auto isIdentitySplat = [&](SDValue Op, bool NSZ) {
17396 if (Op.getOpcode() != ISD::BITCAST ||
17397 Op.getOperand(0).getOpcode() != ARMISD::VMOVIMM)
17398 return false;
17399 uint64_t ImmVal = Op.getOperand(0).getConstantOperandVal(0);
17400 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
17401 return true;
17402 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
17403 return true;
17404 return false;
17405 };
17406
17407 if (Op0.getOpcode() == ISD::VSELECT && Op1.getOpcode() != ISD::VSELECT)
17408 std::swap(Op0, Op1);
17409
17410 if (Op1.getOpcode() != ISD::VSELECT)
17411 return SDValue();
17412
17413 SDNodeFlags FaddFlags = N->getFlags();
17414 bool NSZ = FaddFlags.hasNoSignedZeros();
17415 if (!isIdentitySplat(Op1.getOperand(2), NSZ))
17416 return SDValue();
17417
17418 SDValue FAdd =
17419 DAG.getNode(ISD::FADD, DL, VT, Op0, Op1.getOperand(1), FaddFlags);
17420 return DAG.getNode(ISD::VSELECT, DL, VT, Op1.getOperand(0), FAdd, Op0, FaddFlags);
17421}
17422
17424 SDValue LHS = N->getOperand(0);
17425 SDValue RHS = N->getOperand(1);
17426 EVT VT = N->getValueType(0);
17427 SDLoc DL(N);
17428
17429 if (!N->getFlags().hasAllowReassociation())
17430 return SDValue();
17431
17432 // Combine fadd(a, vcmla(b, c, d)) -> vcmla(fadd(a, b), b, c)
17433 auto ReassocComplex = [&](SDValue A, SDValue B) {
17434 if (A.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
17435 return SDValue();
17436 unsigned Opc = A.getConstantOperandVal(0);
17437 if (Opc != Intrinsic::arm_mve_vcmlaq)
17438 return SDValue();
17439 SDValue VCMLA = DAG.getNode(
17440 ISD::INTRINSIC_WO_CHAIN, DL, VT, A.getOperand(0), A.getOperand(1),
17441 DAG.getNode(ISD::FADD, DL, VT, A.getOperand(2), B, N->getFlags()),
17442 A.getOperand(3), A.getOperand(4));
17443 VCMLA->setFlags(A->getFlags());
17444 return VCMLA;
17445 };
17446 if (SDValue R = ReassocComplex(LHS, RHS))
17447 return R;
17448 if (SDValue R = ReassocComplex(RHS, LHS))
17449 return R;
17450
17451 return SDValue();
17452}
17453
17455 const ARMSubtarget *Subtarget) {
17457 return S;
17458 if (SDValue S = PerformFADDVCMLACombine(N, DAG))
17459 return S;
17460 return SDValue();
17461}
17462
17463/// PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
17464/// can replace combinations of VCVT (integer to floating-point) and VMUL
17465/// when the VMUL has a constant operand that is a power of 2.
17466///
17467/// Example (assume d17 = <float 0.125, float 0.125>):
17468/// vcvt.f32.s32 d16, d16
17469/// vmul.f32 d16, d16, d17
17470/// becomes:
17471/// vcvt.f32.s32 d16, d16, #3
17473 const ARMSubtarget *Subtarget) {
17474 if (!Subtarget->hasNEON())
17475 return SDValue();
17476
17477 SDValue Op = N->getOperand(0);
17478 unsigned OpOpcode = Op.getNode()->getOpcode();
17479 if (!N->getValueType(0).isVector() || !N->getValueType(0).isSimple() ||
17480 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP))
17481 return SDValue();
17482
17483 SDValue ConstVec = N->getOperand(1);
17484 if (!isa<BuildVectorSDNode>(ConstVec))
17485 return SDValue();
17486
17487 MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
17488 uint32_t FloatBits = FloatTy.getSizeInBits();
17489 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
17490 uint32_t IntBits = IntTy.getSizeInBits();
17491 unsigned NumLanes = Op.getValueType().getVectorNumElements();
17492 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17493 // These instructions only exist converting from i32 to f32. We can handle
17494 // smaller integers by generating an extra extend, but larger ones would
17495 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
17496 // these instructions only support v2i32/v4i32 types.
17497 return SDValue();
17498 }
17499
17500 ConstantFPSDNode *CN = isConstOrConstSplatFP(ConstVec, true);
17501 APFloat Recip(0.0f);
17502 if (!CN || !CN->getValueAPF().getExactInverse(&Recip))
17503 return SDValue();
17504
17505 bool IsExact;
17506 APSInt IntVal(33);
17507 if (Recip.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
17508 APFloat::opOK ||
17509 !IsExact)
17510 return SDValue();
17511
17512 int32_t C = IntVal.exactLogBase2();
17513 if (C == -1 || C == 0 || C > 32)
17514 return SDValue();
17515
17516 SDLoc DL(N);
17517 bool isSigned = OpOpcode == ISD::SINT_TO_FP;
17518 SDValue ConvInput = Op.getOperand(0);
17519 if (IntBits < FloatBits)
17521 NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, ConvInput);
17522
17523 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17524 : Intrinsic::arm_neon_vcvtfxu2fp;
17525 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
17526 DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput,
17527 DAG.getConstant(C, DL, MVT::i32));
17528}
17529
17531 const ARMSubtarget *ST) {
17532 if (!ST->hasMVEIntegerOps())
17533 return SDValue();
17534
17535 assert(N->getOpcode() == ISD::VECREDUCE_ADD);
17536 EVT ResVT = N->getValueType(0);
17537 SDValue N0 = N->getOperand(0);
17538 SDLoc dl(N);
17539
17540 // Try to turn vecreduce_add(add(x, y)) into vecreduce(x) + vecreduce(y)
17541 if (ResVT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
17542 (N0.getValueType() == MVT::v4i32 || N0.getValueType() == MVT::v8i16 ||
17543 N0.getValueType() == MVT::v16i8)) {
17544 SDValue Red0 = DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, N0.getOperand(0));
17545 SDValue Red1 = DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, N0.getOperand(1));
17546 return DAG.getNode(ISD::ADD, dl, ResVT, Red0, Red1);
17547 }
17548
17549 // We are looking for something that will have illegal types if left alone,
17550 // but that we can convert to a single instruction under MVE. For example
17551 // vecreduce_add(sext(A, v8i32)) => VADDV.s16 A
17552 // or
17553 // vecreduce_add(mul(zext(A, v16i32), zext(B, v16i32))) => VMLADAV.u8 A, B
17554
17555 // The legal cases are:
17556 // VADDV u/s 8/16/32
17557 // VMLAV u/s 8/16/32
17558 // VADDLV u/s 32
17559 // VMLALV u/s 16/32
17560
17561 // If the input vector is smaller than legal (v4i8/v4i16 for example) we can
17562 // extend it and use v4i32 instead.
17563 auto ExtTypeMatches = [](SDValue A, ArrayRef<MVT> ExtTypes) {
17564 EVT AVT = A.getValueType();
17565 return any_of(ExtTypes, [&](MVT Ty) {
17566 return AVT.getVectorNumElements() == Ty.getVectorNumElements() &&
17567 AVT.bitsLE(Ty);
17568 });
17569 };
17570 auto ExtendIfNeeded = [&](SDValue A, unsigned ExtendCode) {
17571 EVT AVT = A.getValueType();
17572 if (!AVT.is128BitVector())
17573 A = DAG.getNode(
17574 ExtendCode, dl,
17576 *DAG.getContext(),
17578 A);
17579 return A;
17580 };
17581 auto IsVADDV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes) {
17582 if (ResVT != RetTy || N0->getOpcode() != ExtendCode)
17583 return SDValue();
17584 SDValue A = N0->getOperand(0);
17585 if (ExtTypeMatches(A, ExtTypes))
17586 return ExtendIfNeeded(A, ExtendCode);
17587 return SDValue();
17588 };
17589 auto IsPredVADDV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17590 SDValue &Mask) {
17591 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
17593 return SDValue();
17594 Mask = N0->getOperand(0);
17595 SDValue Ext = N0->getOperand(1);
17596 if (Ext->getOpcode() != ExtendCode)
17597 return SDValue();
17598 SDValue A = Ext->getOperand(0);
17599 if (ExtTypeMatches(A, ExtTypes))
17600 return ExtendIfNeeded(A, ExtendCode);
17601 return SDValue();
17602 };
17603 auto IsVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17604 SDValue &A, SDValue &B) {
17605 // For a vmla we are trying to match a larger pattern:
17606 // ExtA = sext/zext A
17607 // ExtB = sext/zext B
17608 // Mul = mul ExtA, ExtB
17609 // vecreduce.add Mul
17610 // There might also be en extra extend between the mul and the addreduce, so
17611 // long as the bitwidth is high enough to make them equivalent (for example
17612 // original v8i16 might be mul at v8i32 and the reduce happens at v8i64).
17613 if (ResVT != RetTy)
17614 return false;
17615 SDValue Mul = N0;
17616 if (Mul->getOpcode() == ExtendCode &&
17617 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17618 ResVT.getScalarSizeInBits())
17619 Mul = Mul->getOperand(0);
17620 if (Mul->getOpcode() != ISD::MUL)
17621 return false;
17622 SDValue ExtA = Mul->getOperand(0);
17623 SDValue ExtB = Mul->getOperand(1);
17624 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17625 return false;
17626 A = ExtA->getOperand(0);
17627 B = ExtB->getOperand(0);
17628 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17629 A = ExtendIfNeeded(A, ExtendCode);
17630 B = ExtendIfNeeded(B, ExtendCode);
17631 return true;
17632 }
17633 return false;
17634 };
17635 auto IsPredVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17636 SDValue &A, SDValue &B, SDValue &Mask) {
17637 // Same as the pattern above with a select for the zero predicated lanes
17638 // ExtA = sext/zext A
17639 // ExtB = sext/zext B
17640 // Mul = mul ExtA, ExtB
17641 // N0 = select Mask, Mul, 0
17642 // vecreduce.add N0
17643 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
17645 return false;
17646 Mask = N0->getOperand(0);
17647 SDValue Mul = N0->getOperand(1);
17648 if (Mul->getOpcode() == ExtendCode &&
17649 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17650 ResVT.getScalarSizeInBits())
17651 Mul = Mul->getOperand(0);
17652 if (Mul->getOpcode() != ISD::MUL)
17653 return false;
17654 SDValue ExtA = Mul->getOperand(0);
17655 SDValue ExtB = Mul->getOperand(1);
17656 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17657 return false;
17658 A = ExtA->getOperand(0);
17659 B = ExtB->getOperand(0);
17660 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17661 A = ExtendIfNeeded(A, ExtendCode);
17662 B = ExtendIfNeeded(B, ExtendCode);
17663 return true;
17664 }
17665 return false;
17666 };
17667 auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
17668 // Split illegal MVT::v16i8->i64 vector reductions into two legal v8i16->i64
17669 // reductions. The operands are extended with MVEEXT, but as they are
17670 // reductions the lane orders do not matter. MVEEXT may be combined with
17671 // loads to produce two extending loads, or else they will be expanded to
17672 // VREV/VMOVL.
17673 EVT VT = Ops[0].getValueType();
17674 if (VT == MVT::v16i8) {
17675 assert((Opcode == ARMISD::VMLALVs || Opcode == ARMISD::VMLALVu) &&
17676 "Unexpected illegal long reduction opcode");
17677 bool IsUnsigned = Opcode == ARMISD::VMLALVu;
17678
17679 SDValue Ext0 =
17680 DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
17681 DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17682 SDValue Ext1 =
17683 DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
17684 DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17685
17686 SDValue MLA0 = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
17687 Ext0, Ext1);
17688 SDValue MLA1 =
17689 DAG.getNode(IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, dl,
17690 DAG.getVTList(MVT::i32, MVT::i32), MLA0, MLA0.getValue(1),
17691 Ext0.getValue(1), Ext1.getValue(1));
17692 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, MLA1, MLA1.getValue(1));
17693 }
17694 SDValue Node = DAG.getNode(Opcode, dl, {MVT::i32, MVT::i32}, Ops);
17695 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Node,
17696 SDValue(Node.getNode(), 1));
17697 };
17698
17699 SDValue A, B;
17700 SDValue Mask;
17701 if (IsVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17702 return DAG.getNode(ARMISD::VMLAVs, dl, ResVT, A, B);
17703 if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17704 return DAG.getNode(ARMISD::VMLAVu, dl, ResVT, A, B);
17705 if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17706 A, B))
17707 return Create64bitNode(ARMISD::VMLALVs, {A, B});
17708 if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17709 A, B))
17710 return Create64bitNode(ARMISD::VMLALVu, {A, B});
17711 if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B))
17712 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17713 DAG.getNode(ARMISD::VMLAVs, dl, MVT::i32, A, B));
17714 if (IsVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B))
17715 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17716 DAG.getNode(ARMISD::VMLAVu, dl, MVT::i32, A, B));
17717
17718 if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17719 Mask))
17720 return DAG.getNode(ARMISD::VMLAVps, dl, ResVT, A, B, Mask);
17721 if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17722 Mask))
17723 return DAG.getNode(ARMISD::VMLAVpu, dl, ResVT, A, B, Mask);
17724 if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17725 Mask))
17726 return Create64bitNode(ARMISD::VMLALVps, {A, B, Mask});
17727 if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17728 Mask))
17729 return Create64bitNode(ARMISD::VMLALVpu, {A, B, Mask});
17730 if (IsPredVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B, Mask))
17731 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17732 DAG.getNode(ARMISD::VMLAVps, dl, MVT::i32, A, B, Mask));
17733 if (IsPredVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B, Mask))
17734 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17735 DAG.getNode(ARMISD::VMLAVpu, dl, MVT::i32, A, B, Mask));
17736
17737 if (SDValue A = IsVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}))
17738 return DAG.getNode(ARMISD::VADDVs, dl, ResVT, A);
17739 if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}))
17740 return DAG.getNode(ARMISD::VADDVu, dl, ResVT, A);
17741 if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}))
17742 return Create64bitNode(ARMISD::VADDLVs, {A});
17743 if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}))
17744 return Create64bitNode(ARMISD::VADDLVu, {A});
17745 if (SDValue A = IsVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}))
17746 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17747 DAG.getNode(ARMISD::VADDVs, dl, MVT::i32, A));
17748 if (SDValue A = IsVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}))
17749 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17750 DAG.getNode(ARMISD::VADDVu, dl, MVT::i32, A));
17751
17752 if (SDValue A = IsPredVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17753 return DAG.getNode(ARMISD::VADDVps, dl, ResVT, A, Mask);
17754 if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17755 return DAG.getNode(ARMISD::VADDVpu, dl, ResVT, A, Mask);
17756 if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}, Mask))
17757 return Create64bitNode(ARMISD::VADDLVps, {A, Mask});
17758 if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}, Mask))
17759 return Create64bitNode(ARMISD::VADDLVpu, {A, Mask});
17760 if (SDValue A = IsPredVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, Mask))
17761 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17762 DAG.getNode(ARMISD::VADDVps, dl, MVT::i32, A, Mask));
17763 if (SDValue A = IsPredVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, Mask))
17764 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17765 DAG.getNode(ARMISD::VADDVpu, dl, MVT::i32, A, Mask));
17766
17767 // Some complications. We can get a case where the two inputs of the mul are
17768 // the same, then the output sext will have been helpfully converted to a
17769 // zext. Turn it back.
17770 SDValue Op = N0;
17771 if (Op->getOpcode() == ISD::VSELECT)
17772 Op = Op->getOperand(1);
17773 if (Op->getOpcode() == ISD::ZERO_EXTEND &&
17774 Op->getOperand(0)->getOpcode() == ISD::MUL) {
17775 SDValue Mul = Op->getOperand(0);
17776 if (Mul->getOperand(0) == Mul->getOperand(1) &&
17777 Mul->getOperand(0)->getOpcode() == ISD::SIGN_EXTEND) {
17778 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, N0->getValueType(0), Mul);
17779 if (Op != N0)
17780 Ext = DAG.getNode(ISD::VSELECT, dl, N0->getValueType(0),
17781 N0->getOperand(0), Ext, N0->getOperand(2));
17782 return DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, Ext);
17783 }
17784 }
17785
17786 return SDValue();
17787}
17788
17789// Looks for vaddv(shuffle) or vmlav(shuffle, shuffle), with a shuffle where all
17790// the lanes are used. Due to the reduction being commutative the shuffle can be
17791// removed.
17793 unsigned VecOp = N->getOperand(0).getValueType().isVector() ? 0 : 2;
17794 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(N->getOperand(VecOp));
17795 if (!Shuf || !Shuf->getOperand(1).isUndef())
17796 return SDValue();
17797
17798 // Check all elements are used once in the mask.
17799 ArrayRef<int> Mask = Shuf->getMask();
17800 APInt SetElts(Mask.size(), 0);
17801 for (int E : Mask) {
17802 if (E < 0 || E >= (int)Mask.size())
17803 return SDValue();
17804 SetElts.setBit(E);
17805 }
17806 if (!SetElts.isAllOnes())
17807 return SDValue();
17808
17809 if (N->getNumOperands() != VecOp + 1) {
17810 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(N->getOperand(VecOp + 1));
17811 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17812 return SDValue();
17813 }
17814
17816 for (SDValue Op : N->ops()) {
17817 if (Op.getValueType().isVector())
17818 Ops.push_back(Op.getOperand(0));
17819 else
17820 Ops.push_back(Op);
17821 }
17822 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), Ops);
17823}
17824
17827 SDValue Op0 = N->getOperand(0);
17828 SDValue Op1 = N->getOperand(1);
17829 unsigned IsTop = N->getConstantOperandVal(2);
17830
17831 // VMOVNT a undef -> a
17832 // VMOVNB a undef -> a
17833 // VMOVNB undef a -> a
17834 if (Op1->isUndef())
17835 return Op0;
17836 if (Op0->isUndef() && !IsTop)
17837 return Op1;
17838
17839 // VMOVNt(c, VQMOVNb(a, b)) => VQMOVNt(c, b)
17840 // VMOVNb(c, VQMOVNb(a, b)) => VQMOVNb(c, b)
17841 if ((Op1->getOpcode() == ARMISD::VQMOVNs ||
17842 Op1->getOpcode() == ARMISD::VQMOVNu) &&
17843 Op1->getConstantOperandVal(2) == 0)
17844 return DCI.DAG.getNode(Op1->getOpcode(), SDLoc(Op1), N->getValueType(0),
17845 Op0, Op1->getOperand(1), N->getOperand(2));
17846
17847 // Only the bottom lanes from Qm (Op1) and either the top or bottom lanes from
17848 // Qd (Op0) are demanded from a VMOVN, depending on whether we are inserting
17849 // into the top or bottom lanes.
17850 unsigned NumElts = N->getValueType(0).getVectorNumElements();
17851 APInt Op1DemandedElts = APInt::getSplat(NumElts, APInt::getLowBitsSet(2, 1));
17852 APInt Op0DemandedElts =
17853 IsTop ? Op1DemandedElts
17854 : APInt::getSplat(NumElts, APInt::getHighBitsSet(2, 1));
17855
17857 if (TLI.SimplifyDemandedVectorElts(Op0, Op0DemandedElts, DCI))
17858 return SDValue(N, 0);
17859 if (TLI.SimplifyDemandedVectorElts(Op1, Op1DemandedElts, DCI))
17860 return SDValue(N, 0);
17861
17862 return SDValue();
17863}
17864
17867 SDValue Op0 = N->getOperand(0);
17868 unsigned IsTop = N->getConstantOperandVal(2);
17869
17870 unsigned NumElts = N->getValueType(0).getVectorNumElements();
17871 APInt Op0DemandedElts =
17872 APInt::getSplat(NumElts, IsTop ? APInt::getLowBitsSet(2, 1)
17873 : APInt::getHighBitsSet(2, 1));
17874
17876 if (TLI.SimplifyDemandedVectorElts(Op0, Op0DemandedElts, DCI))
17877 return SDValue(N, 0);
17878 return SDValue();
17879}
17880
17883 EVT VT = N->getValueType(0);
17884 SDValue LHS = N->getOperand(0);
17885 SDValue RHS = N->getOperand(1);
17886
17887 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(LHS);
17888 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(RHS);
17889 // Turn VQDMULH(shuffle, shuffle) -> shuffle(VQDMULH)
17890 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17891 LHS.getOperand(1).isUndef() && RHS.getOperand(1).isUndef() &&
17892 (LHS.hasOneUse() || RHS.hasOneUse() || LHS == RHS)) {
17893 SDLoc DL(N);
17894 SDValue NewBinOp = DCI.DAG.getNode(N->getOpcode(), DL, VT,
17895 LHS.getOperand(0), RHS.getOperand(0));
17896 SDValue UndefV = LHS.getOperand(1);
17897 return DCI.DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask());
17898 }
17899 return SDValue();
17900}
17901
17903 SDLoc DL(N);
17904 SDValue Op0 = N->getOperand(0);
17905 SDValue Op1 = N->getOperand(1);
17906
17907 // Turn X << -C -> X >> C and viceversa. The negative shifts can come up from
17908 // uses of the intrinsics.
17909 if (auto C = dyn_cast<ConstantSDNode>(N->getOperand(2))) {
17910 int ShiftAmt = C->getSExtValue();
17911 if (ShiftAmt == 0) {
17912 SDValue Merge = DAG.getMergeValues({Op0, Op1}, DL);
17913 DAG.ReplaceAllUsesWith(N, Merge.getNode());
17914 return SDValue();
17915 }
17916
17917 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17918 unsigned NewOpcode =
17919 N->getOpcode() == ARMISD::LSLL ? ARMISD::LSRL : ARMISD::LSLL;
17920 SDValue NewShift = DAG.getNode(NewOpcode, DL, N->getVTList(), Op0, Op1,
17921 DAG.getConstant(-ShiftAmt, DL, MVT::i32));
17922 DAG.ReplaceAllUsesWith(N, NewShift.getNode());
17923 return NewShift;
17924 }
17925 }
17926
17927 return SDValue();
17928}
17929
17930/// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
17932 DAGCombinerInfo &DCI) const {
17933 SelectionDAG &DAG = DCI.DAG;
17934 unsigned IntNo = N->getConstantOperandVal(0);
17935 switch (IntNo) {
17936 default:
17937 // Don't do anything for most intrinsics.
17938 break;
17939
17940 // Vector shifts: check for immediate versions and lower them.
17941 // Note: This is done during DAG combining instead of DAG legalizing because
17942 // the build_vectors for 64-bit vector element shift counts are generally
17943 // not legal, and it is hard to see their values after they get legalized to
17944 // loads from a constant pool.
17945 case Intrinsic::arm_neon_vshifts:
17946 case Intrinsic::arm_neon_vshiftu:
17947 case Intrinsic::arm_neon_vrshifts:
17948 case Intrinsic::arm_neon_vrshiftu:
17949 case Intrinsic::arm_neon_vrshiftn:
17950 case Intrinsic::arm_neon_vqshifts:
17951 case Intrinsic::arm_neon_vqshiftu:
17952 case Intrinsic::arm_neon_vqshiftsu:
17953 case Intrinsic::arm_neon_vqshiftns:
17954 case Intrinsic::arm_neon_vqshiftnu:
17955 case Intrinsic::arm_neon_vqshiftnsu:
17956 case Intrinsic::arm_neon_vqrshiftns:
17957 case Intrinsic::arm_neon_vqrshiftnu:
17958 case Intrinsic::arm_neon_vqrshiftnsu: {
17959 EVT VT = N->getOperand(1).getValueType();
17960 int64_t Cnt;
17961 unsigned VShiftOpc = 0;
17962
17963 switch (IntNo) {
17964 case Intrinsic::arm_neon_vshifts:
17965 case Intrinsic::arm_neon_vshiftu:
17966 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) {
17967 VShiftOpc = ARMISD::VSHLIMM;
17968 break;
17969 }
17970 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) {
17971 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ? ARMISD::VSHRsIMM
17972 : ARMISD::VSHRuIMM);
17973 break;
17974 }
17975 return SDValue();
17976
17977 case Intrinsic::arm_neon_vrshifts:
17978 case Intrinsic::arm_neon_vrshiftu:
17979 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt))
17980 break;
17981 return SDValue();
17982
17983 case Intrinsic::arm_neon_vqshifts:
17984 case Intrinsic::arm_neon_vqshiftu:
17985 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
17986 break;
17987 return SDValue();
17988
17989 case Intrinsic::arm_neon_vqshiftsu:
17990 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
17991 break;
17992 llvm_unreachable("invalid shift count for vqshlu intrinsic");
17993
17994 case Intrinsic::arm_neon_vrshiftn:
17995 case Intrinsic::arm_neon_vqshiftns:
17996 case Intrinsic::arm_neon_vqshiftnu:
17997 case Intrinsic::arm_neon_vqshiftnsu:
17998 case Intrinsic::arm_neon_vqrshiftns:
17999 case Intrinsic::arm_neon_vqrshiftnu:
18000 case Intrinsic::arm_neon_vqrshiftnsu:
18001 // Narrowing shifts require an immediate right shift.
18002 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt))
18003 break;
18004 llvm_unreachable("invalid shift count for narrowing vector shift "
18005 "intrinsic");
18006
18007 default:
18008 llvm_unreachable("unhandled vector shift");
18009 }
18010
18011 switch (IntNo) {
18012 case Intrinsic::arm_neon_vshifts:
18013 case Intrinsic::arm_neon_vshiftu:
18014 // Opcode already set above.
18015 break;
18016 case Intrinsic::arm_neon_vrshifts:
18017 VShiftOpc = ARMISD::VRSHRsIMM;
18018 break;
18019 case Intrinsic::arm_neon_vrshiftu:
18020 VShiftOpc = ARMISD::VRSHRuIMM;
18021 break;
18022 case Intrinsic::arm_neon_vrshiftn:
18023 VShiftOpc = ARMISD::VRSHRNIMM;
18024 break;
18025 case Intrinsic::arm_neon_vqshifts:
18026 VShiftOpc = ARMISD::VQSHLsIMM;
18027 break;
18028 case Intrinsic::arm_neon_vqshiftu:
18029 VShiftOpc = ARMISD::VQSHLuIMM;
18030 break;
18031 case Intrinsic::arm_neon_vqshiftsu:
18032 VShiftOpc = ARMISD::VQSHLsuIMM;
18033 break;
18034 case Intrinsic::arm_neon_vqshiftns:
18035 VShiftOpc = ARMISD::VQSHRNsIMM;
18036 break;
18037 case Intrinsic::arm_neon_vqshiftnu:
18038 VShiftOpc = ARMISD::VQSHRNuIMM;
18039 break;
18040 case Intrinsic::arm_neon_vqshiftnsu:
18041 VShiftOpc = ARMISD::VQSHRNsuIMM;
18042 break;
18043 case Intrinsic::arm_neon_vqrshiftns:
18044 VShiftOpc = ARMISD::VQRSHRNsIMM;
18045 break;
18046 case Intrinsic::arm_neon_vqrshiftnu:
18047 VShiftOpc = ARMISD::VQRSHRNuIMM;
18048 break;
18049 case Intrinsic::arm_neon_vqrshiftnsu:
18050 VShiftOpc = ARMISD::VQRSHRNsuIMM;
18051 break;
18052 }
18053
18054 SDLoc dl(N);
18055 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
18056 N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32));
18057 }
18058
18059 case Intrinsic::arm_neon_vshiftins: {
18060 EVT VT = N->getOperand(1).getValueType();
18061 int64_t Cnt;
18062 unsigned VShiftOpc = 0;
18063
18064 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt))
18065 VShiftOpc = ARMISD::VSLIIMM;
18066 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt))
18067 VShiftOpc = ARMISD::VSRIIMM;
18068 else {
18069 llvm_unreachable("invalid shift count for vsli/vsri intrinsic");
18070 }
18071
18072 SDLoc dl(N);
18073 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
18074 N->getOperand(1), N->getOperand(2),
18075 DAG.getConstant(Cnt, dl, MVT::i32));
18076 }
18077
18078 case Intrinsic::arm_neon_vqrshifts:
18079 case Intrinsic::arm_neon_vqrshiftu:
18080 // No immediate versions of these to check for.
18081 break;
18082
18083 case Intrinsic::arm_neon_vbsl: {
18084 SDLoc dl(N);
18085 return DAG.getNode(ARMISD::VBSP, dl, N->getValueType(0), N->getOperand(1),
18086 N->getOperand(2), N->getOperand(3));
18087 }
18088 case Intrinsic::arm_mve_vqdmlah:
18089 case Intrinsic::arm_mve_vqdmlash:
18090 case Intrinsic::arm_mve_vqrdmlah:
18091 case Intrinsic::arm_mve_vqrdmlash:
18092 case Intrinsic::arm_mve_vmla_n_predicated:
18093 case Intrinsic::arm_mve_vmlas_n_predicated:
18094 case Intrinsic::arm_mve_vqdmlah_predicated:
18095 case Intrinsic::arm_mve_vqdmlash_predicated:
18096 case Intrinsic::arm_mve_vqrdmlah_predicated:
18097 case Intrinsic::arm_mve_vqrdmlash_predicated: {
18098 // These intrinsics all take an i32 scalar operand which is narrowed to the
18099 // size of a single lane of the vector type they return. So we don't need
18100 // any bits of that operand above that point, which allows us to eliminate
18101 // uxth/sxth.
18102 unsigned BitWidth = N->getValueType(0).getScalarSizeInBits();
18103 APInt DemandedMask = APInt::getLowBitsSet(32, BitWidth);
18104 if (SimplifyDemandedBits(N->getOperand(3), DemandedMask, DCI))
18105 return SDValue();
18106 break;
18107 }
18108
18109 case Intrinsic::arm_mve_minv:
18110 case Intrinsic::arm_mve_maxv:
18111 case Intrinsic::arm_mve_minav:
18112 case Intrinsic::arm_mve_maxav:
18113 case Intrinsic::arm_mve_minv_predicated:
18114 case Intrinsic::arm_mve_maxv_predicated:
18115 case Intrinsic::arm_mve_minav_predicated:
18116 case Intrinsic::arm_mve_maxav_predicated: {
18117 // These intrinsics all take an i32 scalar operand which is narrowed to the
18118 // size of a single lane of the vector type they take as the other input.
18119 unsigned BitWidth = N->getOperand(2)->getValueType(0).getScalarSizeInBits();
18120 APInt DemandedMask = APInt::getLowBitsSet(32, BitWidth);
18121 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
18122 return SDValue();
18123 break;
18124 }
18125
18126 case Intrinsic::arm_mve_addv: {
18127 // Turn this intrinsic straight into the appropriate ARMISD::VADDV node,
18128 // which allow PerformADDVecReduce to turn it into VADDLV when possible.
18129 bool Unsigned = N->getConstantOperandVal(2);
18130 unsigned Opc = Unsigned ? ARMISD::VADDVu : ARMISD::VADDVs;
18131 return DAG.getNode(Opc, SDLoc(N), N->getVTList(), N->getOperand(1));
18132 }
18133
18134 case Intrinsic::arm_mve_addlv:
18135 case Intrinsic::arm_mve_addlv_predicated: {
18136 // Same for these, but ARMISD::VADDLV has to be followed by a BUILD_PAIR
18137 // which recombines the two outputs into an i64
18138 bool Unsigned = N->getConstantOperandVal(2);
18139 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
18140 (Unsigned ? ARMISD::VADDLVu : ARMISD::VADDLVs) :
18141 (Unsigned ? ARMISD::VADDLVpu : ARMISD::VADDLVps);
18142
18144 for (unsigned i = 1, e = N->getNumOperands(); i < e; i++)
18145 if (i != 2) // skip the unsigned flag
18146 Ops.push_back(N->getOperand(i));
18147
18148 SDLoc dl(N);
18149 SDValue val = DAG.getNode(Opc, dl, {MVT::i32, MVT::i32}, Ops);
18150 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, val.getValue(0),
18151 val.getValue(1));
18152 }
18153 }
18154
18155 return SDValue();
18156}
18157
18159 EVT VT = Y.getValueType();
18160 if (!VT.isVector())
18161 return hasAndNotCompare(Y);
18162 if (Subtarget->hasMVEIntegerOps())
18163 return VT.is128BitVector();
18164 if (Subtarget->hasNEON())
18165 return VT.is64BitVector() || VT.is128BitVector();
18166 return false;
18167}
18168
18169/// PerformShiftCombine - Checks for immediate versions of vector shifts and
18170/// lowers them. As with the vector shift intrinsics, this is done during DAG
18171/// combining instead of DAG legalizing because the build_vectors for 64-bit
18172/// vector element shift counts are generally not legal, and it is hard to see
18173/// their values after they get legalized to loads from a constant pool.
18176 const ARMSubtarget *ST) {
18177 SelectionDAG &DAG = DCI.DAG;
18178 EVT VT = N->getValueType(0);
18179
18180 if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 &&
18181 N->getOperand(0)->getOpcode() == ISD::AND &&
18182 N->getOperand(0)->hasOneUse()) {
18183 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
18184 return SDValue();
18185 // Look for the pattern (shl (and x, AndMask), ShiftAmt). This doesn't
18186 // usually show up because instcombine prefers to canonicalize it to
18187 // (and (shl x, ShiftAmt) (shl AndMask, ShiftAmt)), but the shift can come
18188 // out of GEP lowering in some cases.
18189 SDValue N0 = N->getOperand(0);
18190 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
18191 if (!ShiftAmtNode)
18192 return SDValue();
18193 uint32_t ShiftAmt = static_cast<uint32_t>(ShiftAmtNode->getZExtValue());
18194 ConstantSDNode *AndMaskNode = dyn_cast<ConstantSDNode>(N0->getOperand(1));
18195 if (!AndMaskNode)
18196 return SDValue();
18197 uint32_t AndMask = static_cast<uint32_t>(AndMaskNode->getZExtValue());
18198 // Don't transform uxtb/uxth.
18199 if (AndMask == 255 || AndMask == 65535)
18200 return SDValue();
18201 if (isMask_32(AndMask)) {
18202 uint32_t MaskedBits = llvm::countl_zero(AndMask);
18203 if (MaskedBits > ShiftAmt) {
18204 SDLoc DL(N);
18205 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
18206 DAG.getConstant(MaskedBits, DL, MVT::i32));
18207 return DAG.getNode(
18208 ISD::SRL, DL, MVT::i32, SHL,
18209 DAG.getConstant(MaskedBits - ShiftAmt, DL, MVT::i32));
18210 }
18211 }
18212 }
18213
18214 // Nothing to be done for scalar shifts.
18216 if (!VT.isVector() || !TLI.isTypeLegal(VT))
18217 return SDValue();
18218 if (ST->hasMVEIntegerOps())
18219 return SDValue();
18220
18221 int64_t Cnt;
18222
18223 switch (N->getOpcode()) {
18224 default: llvm_unreachable("unexpected shift opcode");
18225
18226 case ISD::SHL:
18227 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) {
18228 SDLoc dl(N);
18229 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0),
18230 DAG.getConstant(Cnt, dl, MVT::i32));
18231 }
18232 break;
18233
18234 case ISD::SRA:
18235 case ISD::SRL:
18236 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
18237 unsigned VShiftOpc =
18238 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
18239 SDLoc dl(N);
18240 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
18241 DAG.getConstant(Cnt, dl, MVT::i32));
18242 }
18243 }
18244 return SDValue();
18245}
18246
18247// Look for a sign/zero/fpextend extend of a larger than legal load. This can be
18248// split into multiple extending loads, which are simpler to deal with than an
18249// arbitrary extend. For fp extends we use an integer extending load and a VCVTL
18250// to convert the type to an f32.
18252 SDValue N0 = N->getOperand(0);
18253 if (N0.getOpcode() != ISD::LOAD)
18254 return SDValue();
18256 if (!LD->isSimple() || !N0.hasOneUse() || LD->isIndexed() ||
18257 LD->getExtensionType() != ISD::NON_EXTLOAD)
18258 return SDValue();
18259 EVT FromVT = LD->getValueType(0);
18260 EVT ToVT = N->getValueType(0);
18261 if (!ToVT.isVector())
18262 return SDValue();
18264 EVT ToEltVT = ToVT.getVectorElementType();
18265 EVT FromEltVT = FromVT.getVectorElementType();
18266
18267 unsigned NumElements = 0;
18268 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
18269 NumElements = 4;
18270 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
18271 NumElements = 4;
18272 if (NumElements == 0 ||
18273 (FromEltVT != MVT::f16 && FromVT.getVectorNumElements() == NumElements) ||
18274 FromVT.getVectorNumElements() % NumElements != 0 ||
18275 !isPowerOf2_32(NumElements))
18276 return SDValue();
18277
18278 LLVMContext &C = *DAG.getContext();
18279 SDLoc DL(LD);
18280 // Details about the old load
18281 SDValue Ch = LD->getChain();
18282 SDValue BasePtr = LD->getBasePtr();
18283 Align Alignment = LD->getBaseAlign();
18284 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
18285 AAMDNodes AAInfo = LD->getAAInfo();
18286
18287 ISD::LoadExtType NewExtType =
18288 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
18289 SDValue Offset = DAG.getPOISON(BasePtr.getValueType());
18290 EVT NewFromVT = EVT::getVectorVT(
18291 C, EVT::getIntegerVT(C, FromEltVT.getScalarSizeInBits()), NumElements);
18292 EVT NewToVT = EVT::getVectorVT(
18293 C, EVT::getIntegerVT(C, ToEltVT.getScalarSizeInBits()), NumElements);
18294
18297 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
18298 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
18299 SDValue NewPtr =
18300 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(NewOffset));
18301
18302 SDValue NewLoad =
18303 DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
18304 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18305 Alignment, MMOFlags, AAInfo);
18306 Loads.push_back(NewLoad);
18307 Chains.push_back(SDValue(NewLoad.getNode(), 1));
18308 }
18309
18310 // Float truncs need to extended with VCVTB's into their floating point types.
18311 if (FromEltVT == MVT::f16) {
18313
18314 for (unsigned i = 0; i < Loads.size(); i++) {
18315 SDValue LoadBC =
18316 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v8f16, Loads[i]);
18317 SDValue FPExt = DAG.getNode(ARMISD::VCVTL, DL, MVT::v4f32, LoadBC,
18318 DAG.getConstant(0, DL, MVT::i32));
18319 Extends.push_back(FPExt);
18320 }
18321
18322 Loads = Extends;
18323 }
18324
18325 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
18326 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
18327 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Loads);
18328}
18329
18330/// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
18331/// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
18333 const ARMSubtarget *ST) {
18334 SDValue N0 = N->getOperand(0);
18335 EVT VT = N->getValueType(0);
18336 SDLoc DL(N);
18337
18338 // Check for sign- and zero-extensions of vector extract operations of 8- and
18339 // 16-bit vector elements. NEON and MVE support these directly. They are
18340 // handled during DAG combining because type legalization will promote them
18341 // to 32-bit types and it is messy to recognize the operations after that.
18342 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
18344 SDValue Vec = N0.getOperand(0);
18345 SDValue Lane = N0.getOperand(1);
18346 EVT EltVT = N0.getValueType();
18348
18349 if (VT == MVT::i32 &&
18350 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
18351 TLI.isTypeLegal(Vec.getValueType()) &&
18352 isa<ConstantSDNode>(Lane)) {
18353
18354 unsigned Opc = 0;
18355 switch (N->getOpcode()) {
18356 default: llvm_unreachable("unexpected opcode");
18357 case ISD::SIGN_EXTEND:
18358 Opc = ARMISD::VGETLANEs;
18359 break;
18360 case ISD::ZERO_EXTEND:
18361 case ISD::ANY_EXTEND:
18362 Opc = ARMISD::VGETLANEu;
18363 break;
18364 }
18365 return DAG.getNode(Opc, DL, VT, Vec, Lane);
18366 }
18367 }
18368
18369 if (ST->hasMVEIntegerOps())
18370 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
18371 return NewLoad;
18372
18373 // Combine sext(buildvector(..)) to buildvector(sext(..)) to help avoid
18374 // difficult to lower i1 buildvector.
18375 if (ST->hasMVEIntegerOps() && N0.getValueType().getScalarSizeInBits() == 1 &&
18376 N0.getOpcode() == ISD::BUILD_VECTOR && VT.getScalarSizeInBits() <= 32) {
18378 for (unsigned I = 0; I < N0.getNumOperands(); I++) {
18379 SDValue InReg = N0.getOperand(I);
18380 if (N->getOpcode() == ISD::ZERO_EXTEND)
18381 InReg = DAG.getNode(ISD::AND, DL, InReg.getValueType(), InReg,
18382 DAG.getConstant(1, DL, InReg.getValueType()));
18383 else if (N->getOpcode() == ISD::SIGN_EXTEND)
18384 InReg = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, InReg.getValueType(),
18385 InReg, DAG.getValueType(MVT::i1));
18386 SDValue Ext = DAG.getNode(N->getOpcode(), DL, MVT::i32, InReg);
18387 Ops.push_back(Ext);
18388 }
18389 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
18390 }
18391
18392 return SDValue();
18393}
18394
18396 const ARMSubtarget *ST) {
18397 if (ST->hasMVEFloatOps())
18398 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
18399 return NewLoad;
18400
18401 return SDValue();
18402}
18403
18404// Lower smin(smax(x, C1), C2) to ssat or usat, if they have saturating
18405// constant bounds.
18407 const ARMSubtarget *Subtarget) {
18408 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
18409 !Subtarget->isThumb2())
18410 return SDValue();
18411
18412 EVT VT = Op.getValueType();
18413 SDValue Op0 = Op.getOperand(0);
18414
18415 if (VT != MVT::i32 ||
18416 (Op0.getOpcode() != ISD::SMIN && Op0.getOpcode() != ISD::SMAX) ||
18417 !isa<ConstantSDNode>(Op.getOperand(1)) ||
18419 return SDValue();
18420
18421 SDValue Min = Op;
18422 SDValue Max = Op0;
18423 SDValue Input = Op0.getOperand(0);
18424 if (Min.getOpcode() == ISD::SMAX)
18425 std::swap(Min, Max);
18426
18427 if (Min.getOpcode() != ISD::SMIN || Max.getOpcode() != ISD::SMAX)
18428 return SDValue();
18429
18430 APInt MinC = Min.getConstantOperandAPInt(1);
18431 APInt MaxC = Max.getConstantOperandAPInt(1);
18432 if (MaxC.sgt(MinC))
18433 return SDValue();
18434
18435 SDLoc DL(Op);
18436
18437 // A clamp whose bounds are already a saturation range maps to a single
18438 // SSAT / USAT.
18439 if ((MinC + 1).isPowerOf2()) {
18440 if (MinC == ~MaxC)
18441 return DAG.getNode(ARMISD::SSAT, DL, VT, Input,
18442 DAG.getConstant(MinC.countr_one(), DL, VT));
18443 if (MaxC == 0)
18444 return DAG.getNode(ARMISD::USAT, DL, VT, Input,
18445 DAG.getConstant(MinC.countr_one(), DL, VT));
18446 }
18447
18448 // For power-of-two clamp widths, convert the range to be zero-centered,
18449 // apply SSAT, and convert the result back.
18450 //
18451 // Width = Hi - Lo + 1
18452 // Center = Lo + Width / 2
18453 // Result = ssat(X - Center) + Center
18454 //
18455 // The idea is to shift the input so that the clamp range is centered
18456 // around zero, apply ssat, and then shift the result back.
18457 //
18458 // For example clamp(X, -118, 137) -> Width = 256, Center = 10, so it becomes
18459 // ssat(X - 10, 8) + 10
18460
18461 APInt Width = MinC - MaxC + 1;
18462 if (!Width.isPowerOf2() || Width.isOne())
18463 return SDValue();
18464 unsigned SatBit = Width.logBase2() - 1; // ssat to SatBit + 1 signed bits
18465 APInt Center = MaxC + Width.lshr(1);
18466
18467 // The rewrite is only valid when X - Center does not overflow;
18468 SDValue NegC = DAG.getConstant(-Center, DL, VT);
18470 return SDValue();
18471
18472 SDValue Shifted = DAG.getNode(ISD::ADD, DL, VT, Input, NegC);
18473 SDValue Sat = DAG.getNode(ARMISD::SSAT, DL, VT, Shifted,
18474 DAG.getConstant(SatBit, DL, VT));
18475 return DAG.getNode(ISD::ADD, DL, VT, Sat, DAG.getConstant(Center, DL, VT));
18476}
18477
18478/// PerformMinMaxCombine - Target-specific DAG combining for creating truncating
18479/// saturates.
18481 const ARMSubtarget *ST) {
18482 EVT VT = N->getValueType(0);
18483 SDValue N0 = N->getOperand(0);
18484
18485 if (VT == MVT::i32)
18486 return PerformMinMaxToSatCombine(SDValue(N, 0), DAG, ST);
18487
18488 if (!ST->hasMVEIntegerOps())
18489 return SDValue();
18490
18491 if (SDValue V = PerformVQDMULHCombine(N, DAG))
18492 return V;
18493
18494 if (VT != MVT::v4i32 && VT != MVT::v8i16)
18495 return SDValue();
18496
18497 auto IsSignedSaturate = [&](SDNode *Min, SDNode *Max) {
18498 // Check one is a smin and the other is a smax
18499 if (Min->getOpcode() != ISD::SMIN)
18500 std::swap(Min, Max);
18501 if (Min->getOpcode() != ISD::SMIN || Max->getOpcode() != ISD::SMAX)
18502 return false;
18503
18504 APInt SaturateC;
18505 if (VT == MVT::v4i32)
18506 SaturateC = APInt(32, (1 << 15) - 1, true);
18507 else //if (VT == MVT::v8i16)
18508 SaturateC = APInt(16, (1 << 7) - 1, true);
18509
18510 APInt MinC, MaxC;
18511 if (!ISD::isConstantSplatVector(Min->getOperand(1), MinC) ||
18512 MinC != SaturateC)
18513 return false;
18514 if (!ISD::isConstantSplatVector(Max->getOperand(1), MaxC) ||
18515 MaxC != ~SaturateC)
18516 return false;
18517 return true;
18518 };
18519
18520 if (IsSignedSaturate(N, N0.getNode())) {
18521 SDLoc DL(N);
18522 MVT ExtVT, HalfVT;
18523 if (VT == MVT::v4i32) {
18524 HalfVT = MVT::v8i16;
18525 ExtVT = MVT::v4i16;
18526 } else { // if (VT == MVT::v8i16)
18527 HalfVT = MVT::v16i8;
18528 ExtVT = MVT::v8i8;
18529 }
18530
18531 // Create a VQMOVNB with undef top lanes, then signed extended into the top
18532 // half. That extend will hopefully be removed if only the bottom bits are
18533 // demanded (though a truncating store, for example).
18534 SDValue VQMOVN =
18535 DAG.getNode(ARMISD::VQMOVNs, DL, HalfVT, DAG.getUNDEF(HalfVT),
18536 N0->getOperand(0), DAG.getConstant(0, DL, MVT::i32));
18537 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
18538 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Bitcast,
18539 DAG.getValueType(ExtVT));
18540 }
18541
18542 auto IsUnsignedSaturate = [&](SDNode *Min) {
18543 // For unsigned, we just need to check for <= 0xffff
18544 if (Min->getOpcode() != ISD::UMIN)
18545 return false;
18546
18547 APInt SaturateC;
18548 if (VT == MVT::v4i32)
18549 SaturateC = APInt(32, (1 << 16) - 1, true);
18550 else //if (VT == MVT::v8i16)
18551 SaturateC = APInt(16, (1 << 8) - 1, true);
18552
18553 APInt MinC;
18554 if (!ISD::isConstantSplatVector(Min->getOperand(1), MinC) ||
18555 MinC != SaturateC)
18556 return false;
18557 return true;
18558 };
18559
18560 if (IsUnsignedSaturate(N)) {
18561 SDLoc DL(N);
18562 MVT HalfVT;
18563 unsigned ExtConst;
18564 if (VT == MVT::v4i32) {
18565 HalfVT = MVT::v8i16;
18566 ExtConst = 0x0000FFFF;
18567 } else { //if (VT == MVT::v8i16)
18568 HalfVT = MVT::v16i8;
18569 ExtConst = 0x00FF;
18570 }
18571
18572 // Create a VQMOVNB with undef top lanes, then ZExt into the top half with
18573 // an AND. That extend will hopefully be removed if only the bottom bits are
18574 // demanded (though a truncating store, for example).
18575 SDValue VQMOVN =
18576 DAG.getNode(ARMISD::VQMOVNu, DL, HalfVT, DAG.getUNDEF(HalfVT), N0,
18577 DAG.getConstant(0, DL, MVT::i32));
18578 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
18579 return DAG.getNode(ISD::AND, DL, VT, Bitcast,
18580 DAG.getConstant(ExtConst, DL, VT));
18581 }
18582
18583 return SDValue();
18584}
18585
18588 if (!C)
18589 return nullptr;
18590 const APInt *CV = &C->getAPIntValue();
18591 return CV->isPowerOf2() ? CV : nullptr;
18592}
18593
18595 // If we have a CMOV, OR and AND combination such as:
18596 // if (x & CN)
18597 // y |= CM;
18598 //
18599 // And:
18600 // * CN is a single bit;
18601 // * All bits covered by CM are known zero in y
18602 //
18603 // Then we can convert this into a sequence of BFI instructions. This will
18604 // always be a win if CM is a single bit, will always be no worse than the
18605 // TST&OR sequence if CM is two bits, and for thumb will be no worse if CM is
18606 // three bits (due to the extra IT instruction).
18607
18608 SDValue Op0 = CMOV->getOperand(0);
18609 SDValue Op1 = CMOV->getOperand(1);
18610 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
18611 SDValue CmpZ = CMOV->getOperand(3);
18612
18613 // The compare must be against zero.
18614 if (!isNullConstant(CmpZ->getOperand(1)))
18615 return SDValue();
18616
18617 assert(CmpZ->getOpcode() == ARMISD::CMPZ);
18618 SDValue And = CmpZ->getOperand(0);
18619 if (And->getOpcode() != ISD::AND)
18620 return SDValue();
18621 const APInt *AndC = isPowerOf2Constant(And->getOperand(1));
18622 if (!AndC)
18623 return SDValue();
18624 SDValue X = And->getOperand(0);
18625
18626 if (CC == ARMCC::EQ) {
18627 // We're performing an "equal to zero" compare. Swap the operands so we
18628 // canonicalize on a "not equal to zero" compare.
18629 std::swap(Op0, Op1);
18630 } else {
18631 assert(CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?");
18632 }
18633
18634 if (Op1->getOpcode() != ISD::OR)
18635 return SDValue();
18636
18638 if (!OrC)
18639 return SDValue();
18640 SDValue Y = Op1->getOperand(0);
18641
18642 if (Op0 != Y)
18643 return SDValue();
18644
18645 // Now, is it profitable to continue?
18646 APInt OrCI = OrC->getAPIntValue();
18647 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18648 if (OrCI.popcount() > Heuristic)
18649 return SDValue();
18650
18651 // Lastly, can we determine that the bits defined by OrCI
18652 // are zero in Y?
18654 if ((OrCI & Known.Zero) != OrCI)
18655 return SDValue();
18656
18657 // OK, we can do the combine.
18658 SDValue V = Y;
18659 SDLoc dl(X);
18660 EVT VT = X.getValueType();
18661 unsigned BitInX = AndC->logBase2();
18662
18663 if (BitInX != 0) {
18664 // We must shift X first.
18665 X = DAG.getNode(ISD::SRL, dl, VT, X,
18666 DAG.getConstant(BitInX, dl, VT));
18667 }
18668
18669 for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits();
18670 BitInY < NumActiveBits; ++BitInY) {
18671 if (OrCI[BitInY] == 0)
18672 continue;
18673 APInt Mask(VT.getSizeInBits(), 0);
18674 Mask.setBit(BitInY);
18675 V = DAG.getNode(ARMISD::BFI, dl, VT, V, X,
18676 // Confusingly, the operand is an *inverted* mask.
18677 DAG.getConstant(~Mask, dl, VT));
18678 }
18679
18680 return V;
18681}
18682
18683// Given N, the value controlling the conditional branch, search for the loop
18684// intrinsic, returning it, along with how the value is used. We need to handle
18685// patterns such as the following:
18686// (brcond (xor (setcc (loop.decrement), 0, ne), 1), exit)
18687// (brcond (setcc (loop.decrement), 0, eq), exit)
18688// (brcond (setcc (loop.decrement), 0, ne), header)
18690 bool &Negate) {
18691 switch (N->getOpcode()) {
18692 default:
18693 break;
18694 case ISD::XOR: {
18695 if (!isa<ConstantSDNode>(N.getOperand(1)))
18696 return SDValue();
18697 if (!cast<ConstantSDNode>(N.getOperand(1))->isOne())
18698 return SDValue();
18699 Negate = !Negate;
18700 return SearchLoopIntrinsic(N.getOperand(0), CC, Imm, Negate);
18701 }
18702 case ISD::SETCC: {
18703 auto *Const = dyn_cast<ConstantSDNode>(N.getOperand(1));
18704 if (!Const)
18705 return SDValue();
18706 if (Const->isZero())
18707 Imm = 0;
18708 else if (Const->isOne())
18709 Imm = 1;
18710 else
18711 return SDValue();
18712 CC = N.getOperandCondCode(2);
18713 return SearchLoopIntrinsic(N->getOperand(0), CC, Imm, Negate);
18714 }
18716 unsigned IntOp = N.getConstantOperandVal(1);
18717 if (IntOp != Intrinsic::test_start_loop_iterations &&
18718 IntOp != Intrinsic::loop_decrement_reg)
18719 return SDValue();
18720 return N;
18721 }
18722 }
18723 return SDValue();
18724}
18725
18728 const ARMSubtarget *ST) {
18729
18730 // The hwloop intrinsics that we're interested are used for control-flow,
18731 // either for entering or exiting the loop:
18732 // - test.start.loop.iterations will test whether its operand is zero. If it
18733 // is zero, the proceeding branch should not enter the loop.
18734 // - loop.decrement.reg also tests whether its operand is zero. If it is
18735 // zero, the proceeding branch should not branch back to the beginning of
18736 // the loop.
18737 // So here, we need to check that how the brcond is using the result of each
18738 // of the intrinsics to ensure that we're branching to the right place at the
18739 // right time.
18740
18742 SDValue Cond;
18743 int Imm = 1;
18744 bool Negate = false;
18745 SDValue Chain = N->getOperand(0);
18746 SDValue Dest;
18747
18748 if (N->getOpcode() == ISD::BRCOND) {
18749 CC = ISD::SETEQ;
18750 Cond = N->getOperand(1);
18751 Dest = N->getOperand(2);
18752 } else {
18753 assert(N->getOpcode() == ISD::BR_CC && "Expected BRCOND or BR_CC!");
18754 CC = N->getOperandCondCode(1);
18755 Cond = N->getOperand(2);
18756 Dest = N->getOperand(4);
18757 if (auto *Const = dyn_cast<ConstantSDNode>(N->getOperand(3))) {
18758 if (!Const->isOne() && !Const->isZero())
18759 return SDValue();
18760 Imm = Const->getZExtValue();
18761 } else
18762 return SDValue();
18763 }
18764
18766 if (!Int)
18767 return SDValue();
18768
18769 if (Negate)
18770 CC = ISD::getSetCCInverse(CC, /* Integer inverse */ MVT::i32);
18771
18772 auto IsTrueIfZero = [](ISD::CondCode CC, int Imm) {
18773 return (CC == ISD::SETEQ && Imm == 0) ||
18774 (CC == ISD::SETNE && Imm == 1) ||
18775 (CC == ISD::SETLT && Imm == 1) ||
18776 (CC == ISD::SETULT && Imm == 1);
18777 };
18778
18779 auto IsFalseIfZero = [](ISD::CondCode CC, int Imm) {
18780 return (CC == ISD::SETEQ && Imm == 1) ||
18781 (CC == ISD::SETNE && Imm == 0) ||
18782 (CC == ISD::SETGT && Imm == 0) ||
18783 (CC == ISD::SETUGT && Imm == 0) ||
18784 (CC == ISD::SETGE && Imm == 1) ||
18785 (CC == ISD::SETUGE && Imm == 1);
18786 };
18787
18788 assert((IsTrueIfZero(CC, Imm) || IsFalseIfZero(CC, Imm)) &&
18789 "unsupported condition");
18790
18791 SDLoc dl(Int);
18792 SelectionDAG &DAG = DCI.DAG;
18793 SDValue Elements = Int.getOperand(2);
18794 unsigned IntOp = Int->getConstantOperandVal(1);
18795 assert((N->hasOneUse() && N->user_begin()->getOpcode() == ISD::BR) &&
18796 "expected single br user");
18797 SDNode *Br = *N->user_begin();
18798 SDValue OtherTarget = Br->getOperand(1);
18799
18800 // Update the unconditional branch to branch to the given Dest.
18801 auto UpdateUncondBr = [](SDNode *Br, SDValue Dest, SelectionDAG &DAG) {
18802 SDValue NewBrOps[] = { Br->getOperand(0), Dest };
18803 SDValue NewBr = DAG.getNode(ISD::BR, SDLoc(Br), MVT::Other, NewBrOps);
18804 DAG.ReplaceAllUsesOfValueWith(SDValue(Br, 0), NewBr);
18805 };
18806
18807 if (IntOp == Intrinsic::test_start_loop_iterations) {
18808 SDValue Res;
18809 SDValue Setup = DAG.getNode(ARMISD::WLSSETUP, dl, MVT::i32, Elements);
18810 // We expect this 'instruction' to branch when the counter is zero.
18811 if (IsTrueIfZero(CC, Imm)) {
18812 SDValue Ops[] = {Chain, Setup, Dest};
18813 Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops);
18814 } else {
18815 // The logic is the reverse of what we need for WLS, so find the other
18816 // basic block target: the target of the proceeding br.
18817 UpdateUncondBr(Br, Dest, DAG);
18818
18819 SDValue Ops[] = {Chain, Setup, OtherTarget};
18820 Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops);
18821 }
18822 // Update LR count to the new value
18823 DAG.ReplaceAllUsesOfValueWith(Int.getValue(0), Setup);
18824 // Update chain
18825 DAG.ReplaceAllUsesOfValueWith(Int.getValue(2), Int.getOperand(0));
18826 return Res;
18827 } else {
18828 SDValue Size =
18829 DAG.getTargetConstant(Int.getConstantOperandVal(3), dl, MVT::i32);
18830 SDValue Args[] = { Int.getOperand(0), Elements, Size, };
18831 SDValue LoopDec = DAG.getNode(ARMISD::LOOP_DEC, dl,
18832 DAG.getVTList(MVT::i32, MVT::Other), Args);
18833 DAG.ReplaceAllUsesWith(Int.getNode(), LoopDec.getNode());
18834
18835 // We expect this instruction to branch when the count is not zero.
18836 SDValue Target = IsFalseIfZero(CC, Imm) ? Dest : OtherTarget;
18837
18838 // Update the unconditional branch to target the loop preheader if we've
18839 // found the condition has been reversed.
18840 if (Target == OtherTarget)
18841 UpdateUncondBr(Br, Dest, DAG);
18842
18843 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
18844 SDValue(LoopDec.getNode(), 1), Chain);
18845
18846 SDValue EndArgs[] = { Chain, SDValue(LoopDec.getNode(), 0), Target };
18847 return DAG.getNode(ARMISD::LE, dl, MVT::Other, EndArgs);
18848 }
18849 return SDValue();
18850}
18851
18852/// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
18853SDValue
18855 SDValue Cmp = N->getOperand(3);
18856 if (Cmp.getOpcode() != ARMISD::CMPZ)
18857 // Only looking at NE cases.
18858 return SDValue();
18859
18860 SDLoc dl(N);
18861 SDValue LHS = Cmp.getOperand(0);
18862 SDValue RHS = Cmp.getOperand(1);
18863 SDValue Chain = N->getOperand(0);
18864 SDValue BB = N->getOperand(1);
18865 SDValue ARMcc = N->getOperand(2);
18867
18868 // (brcond Chain BB ne (cmpz (and (cmov 0 1 CC Flags) 1) 0))
18869 // -> (brcond Chain BB CC Flags)
18870 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() &&
18871 LHS->getOperand(0)->getOpcode() == ARMISD::CMOV &&
18872 LHS->getOperand(0)->hasOneUse() &&
18873 isNullConstant(LHS->getOperand(0)->getOperand(0)) &&
18874 isOneConstant(LHS->getOperand(0)->getOperand(1)) &&
18875 isOneConstant(LHS->getOperand(1)) && isNullConstant(RHS)) {
18876 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, BB,
18877 LHS->getOperand(0)->getOperand(2),
18878 LHS->getOperand(0)->getOperand(3));
18879 }
18880
18881 return SDValue();
18882}
18883
18884/// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
18885SDValue
18887 SDLoc dl(N);
18888 EVT VT = N->getValueType(0);
18889 SDValue FalseVal = N->getOperand(0);
18890 SDValue TrueVal = N->getOperand(1);
18891 SDValue ARMcc = N->getOperand(2);
18892 SDValue Cmp = N->getOperand(3);
18893
18894 // Try to form CSINV etc.
18895 unsigned Opcode;
18896 bool InvertCond;
18897 if (SDValue CSetOp =
18898 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
18899 if (InvertCond) {
18900 ARMCC::CondCodes CondCode =
18901 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue();
18902 CondCode = ARMCC::getOppositeCondition(CondCode);
18903 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
18904 }
18905 return DAG.getNode(Opcode, dl, VT, CSetOp, CSetOp, ARMcc, Cmp);
18906 }
18907
18908 if (Cmp.getOpcode() != ARMISD::CMPZ)
18909 // Only looking at EQ and NE cases.
18910 return SDValue();
18911
18912 SDValue LHS = Cmp.getOperand(0);
18913 SDValue RHS = Cmp.getOperand(1);
18915
18916 // BFI is only available on V6T2+.
18917 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) {
18919 if (R)
18920 return R;
18921 }
18922
18923 // Simplify
18924 // mov r1, r0
18925 // cmp r1, x
18926 // mov r0, y
18927 // moveq r0, x
18928 // to
18929 // cmp r0, x
18930 // movne r0, y
18931 //
18932 // mov r1, r0
18933 // cmp r1, x
18934 // mov r0, x
18935 // movne r0, y
18936 // to
18937 // cmp r0, x
18938 // movne r0, y
18939 /// FIXME: Turn this into a target neutral optimization?
18940 SDValue Res;
18941 if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
18942 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc, Cmp);
18943 } else if (CC == ARMCC::EQ && TrueVal == RHS) {
18944 SDValue ARMcc;
18945 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
18946 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc, NewCmp);
18947 }
18948
18949 // (cmov F T ne (cmpz (cmov 0 1 CC Flags) 0))
18950 // -> (cmov F T CC Flags)
18951 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse() &&
18952 isNullConstant(LHS->getOperand(0)) && isOneConstant(LHS->getOperand(1)) &&
18953 isNullConstant(RHS)) {
18954 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal,
18955 LHS->getOperand(2), LHS->getOperand(3));
18956 }
18957
18958 if (!VT.isInteger())
18959 return SDValue();
18960
18961 // Fold away an unnecessary CMPZ/CMOV
18962 // CMOV A, B, C1, (CMPZ (CMOV 1, 0, C2, D), 0) ->
18963 // if C1==EQ -> CMOV A, B, C2, D
18964 // if C1==NE -> CMOV A, B, NOT(C2), D
18965 if (N->getConstantOperandVal(2) == ARMCC::EQ ||
18966 N->getConstantOperandVal(2) == ARMCC::NE) {
18968 if (SDValue C = IsCMPZCSINC(N->getOperand(3).getNode(), Cond)) {
18969 if (N->getConstantOperandVal(2) == ARMCC::NE)
18971 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
18972 N->getOperand(1),
18973 DAG.getConstant(Cond, SDLoc(N), MVT::i32), C);
18974 }
18975 }
18976
18977 // Materialize a boolean comparison for integers so we can avoid branching.
18978 if (isNullConstant(FalseVal)) {
18979 if (CC == ARMCC::EQ && isOneConstant(TrueVal)) {
18980 if (!Subtarget->isThumb1Only() && Subtarget->hasV5TOps()) {
18981 // If x == y then x - y == 0 and ARM's CLZ will return 32, shifting it
18982 // right 5 bits will make that 32 be 1, otherwise it will be 0.
18983 // CMOV 0, 1, ==, (CMPZ x, y) -> SRL (CTLZ (SUB x, y)), 5
18984 SDValue Value;
18985 if (VT == MVT::i32 && Subtarget->isThumb2() && isNullConstant(RHS) &&
18986 Cmp.hasOneUse())
18987 Value = getShiftedLowMaskZeroTest(LHS, DAG, dl);
18988 if (!Value)
18989 Value = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
18990 Res =
18991 DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::CTLZ, dl, VT, Value),
18992 DAG.getConstant(5, dl, MVT::i32));
18993 } else {
18994 // CMOV 0, 1, ==, (CMPZ x, y) ->
18995 // (UADDO_CARRY (SUB x, y), t:0, t:1)
18996 // where t = (USUBO_CARRY 0, (SUB x, y), 0)
18997 //
18998 // The USUBO_CARRY computes 0 - (x - y) and this will give a borrow when
18999 // x != y. In other words, a carry C == 1 when x == y, C == 0
19000 // otherwise.
19001 // The final UADDO_CARRY computes
19002 // x - y + (0 - (x - y)) + C == C
19003 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
19004 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
19005 SDValue Neg = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, Sub);
19006 // ISD::USUBO_CARRY returns a borrow but we want the carry here
19007 // actually.
19008 SDValue Carry =
19009 DAG.getNode(ISD::SUB, dl, MVT::i32,
19010 DAG.getConstant(1, dl, MVT::i32), Neg.getValue(1));
19011 Res = DAG.getNode(ISD::UADDO_CARRY, dl, VTs, Sub, Neg, Carry);
19012 }
19013 } else if (CC == ARMCC::NE && !isNullConstant(RHS) &&
19014 (!Subtarget->isThumb1Only() || isPowerOf2Constant(TrueVal))) {
19015 // This seems pointless but will allow us to combine it further below.
19016 // CMOV 0, z, !=, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
19017 SDValue Sub =
19018 DAG.getNode(ARMISD::SUBC, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
19019 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, TrueVal, ARMcc,
19020 Sub.getValue(1));
19021 FalseVal = Sub;
19022 }
19023 } else if (isNullConstant(TrueVal)) {
19024 if (CC == ARMCC::EQ && !isNullConstant(RHS) &&
19025 (!Subtarget->isThumb1Only() || isPowerOf2Constant(FalseVal))) {
19026 // This seems pointless but will allow us to combine it further below
19027 // Note that we change == for != as this is the dual for the case above.
19028 // CMOV z, 0, ==, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
19029 SDValue Sub =
19030 DAG.getNode(ARMISD::SUBC, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
19031 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, FalseVal,
19032 DAG.getConstant(ARMCC::NE, dl, MVT::i32),
19033 Sub.getValue(1));
19034 FalseVal = Sub;
19035 }
19036 }
19037
19038 // On Thumb1, the DAG above may be further combined if z is a power of 2
19039 // (z == 2 ^ K).
19040 // CMOV (SUBC x, y), z, !=, (SUBC x, y):1 ->
19041 // t1 = (USUBO (SUB x, y), 1)
19042 // t2 = (USUBO_CARRY (SUB x, y), t1:0, t1:1)
19043 // Result = if K != 0 then (SHL t2:0, K) else t2:0
19044 //
19045 // This also handles the special case of comparing against zero; it's
19046 // essentially, the same pattern, except there's no SUBC:
19047 // CMOV x, z, !=, (CMPZ x, 0) ->
19048 // t1 = (USUBO x, 1)
19049 // t2 = (USUBO_CARRY x, t1:0, t1:1)
19050 // Result = if K != 0 then (SHL t2:0, K) else t2:0
19051 const APInt *TrueConst;
19052 if (Subtarget->isThumb1Only() && CC == ARMCC::NE &&
19053 ((FalseVal.getOpcode() == ARMISD::SUBC && FalseVal.getOperand(0) == LHS &&
19054 FalseVal.getOperand(1) == RHS) ||
19055 (FalseVal == LHS && isNullConstant(RHS))) &&
19056 (TrueConst = isPowerOf2Constant(TrueVal))) {
19057 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
19058 unsigned ShiftAmount = TrueConst->logBase2();
19059 if (ShiftAmount)
19060 TrueVal = DAG.getConstant(1, dl, VT);
19061 SDValue Subc = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, TrueVal);
19062 Res = DAG.getNode(ISD::USUBO_CARRY, dl, VTs, FalseVal, Subc,
19063 Subc.getValue(1));
19064
19065 if (ShiftAmount)
19066 Res = DAG.getNode(ISD::SHL, dl, VT, Res,
19067 DAG.getConstant(ShiftAmount, dl, MVT::i32));
19068 }
19069
19070 if (Res.getNode()) {
19072 // Capture demanded bits information that would be otherwise lost.
19073 if (Known.Zero == 0xfffffffe)
19074 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
19075 DAG.getValueType(MVT::i1));
19076 else if (Known.Zero == 0xffffff00)
19077 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
19078 DAG.getValueType(MVT::i8));
19079 else if (Known.Zero == 0xffff0000)
19080 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
19081 DAG.getValueType(MVT::i16));
19082 }
19083
19084 return Res;
19085}
19086
19089 const ARMSubtarget *ST) {
19090 SelectionDAG &DAG = DCI.DAG;
19091 SDValue Src = N->getOperand(0);
19092 EVT DstVT = N->getValueType(0);
19093
19094 // Convert v4f32 bitcast (v4i32 vdup (i32)) -> v4f32 vdup (i32) under MVE.
19095 if (ST->hasMVEIntegerOps() && Src.getOpcode() == ARMISD::VDUP) {
19096 EVT SrcVT = Src.getValueType();
19097 if (SrcVT.getScalarSizeInBits() == DstVT.getScalarSizeInBits())
19098 return DAG.getNode(ARMISD::VDUP, SDLoc(N), DstVT, Src.getOperand(0));
19099 }
19100
19101 // We may have a bitcast of something that has already had this bitcast
19102 // combine performed on it, so skip past any VECTOR_REG_CASTs.
19103 if (Src.getOpcode() == ARMISD::VECTOR_REG_CAST &&
19104 Src.getOperand(0).getValueType().getScalarSizeInBits() <=
19105 Src.getValueType().getScalarSizeInBits())
19106 Src = Src.getOperand(0);
19107
19108 // Bitcast from element-wise VMOV or VMVN doesn't need VREV if the VREV that
19109 // would be generated is at least the width of the element type.
19110 EVT SrcVT = Src.getValueType();
19111 if ((Src.getOpcode() == ARMISD::VMOVIMM ||
19112 Src.getOpcode() == ARMISD::VMVNIMM ||
19113 Src.getOpcode() == ARMISD::VMOVFPIMM) &&
19114 SrcVT.getScalarSizeInBits() <= DstVT.getScalarSizeInBits() &&
19115 DAG.getDataLayout().isBigEndian())
19116 return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(N), DstVT, Src);
19117
19118 // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD x
19119 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
19120 return R;
19121
19122 return SDValue();
19123}
19124
19125// Some combines for the MVETrunc truncations legalizer helper. Also lowers the
19126// node into stack operations after legalizeOps.
19129 SelectionDAG &DAG = DCI.DAG;
19130 EVT VT = N->getValueType(0);
19131 SDLoc DL(N);
19132
19133 // MVETrunc(Undef, Undef) -> Undef
19134 if (all_of(N->ops(), [](SDValue Op) { return Op.isUndef(); }))
19135 return DAG.getUNDEF(VT);
19136
19137 // MVETrunc(MVETrunc a b, MVETrunc c, d) -> MVETrunc
19138 if (N->getNumOperands() == 2 &&
19139 N->getOperand(0).getOpcode() == ARMISD::MVETRUNC &&
19140 N->getOperand(1).getOpcode() == ARMISD::MVETRUNC)
19141 return DAG.getNode(ARMISD::MVETRUNC, DL, VT, N->getOperand(0).getOperand(0),
19142 N->getOperand(0).getOperand(1),
19143 N->getOperand(1).getOperand(0),
19144 N->getOperand(1).getOperand(1));
19145
19146 // MVETrunc(shuffle, shuffle) -> VMOVN
19147 if (N->getNumOperands() == 2 &&
19148 N->getOperand(0).getOpcode() == ISD::VECTOR_SHUFFLE &&
19149 N->getOperand(1).getOpcode() == ISD::VECTOR_SHUFFLE) {
19150 auto *S0 = cast<ShuffleVectorSDNode>(N->getOperand(0).getNode());
19151 auto *S1 = cast<ShuffleVectorSDNode>(N->getOperand(1).getNode());
19152
19153 if (S0->getOperand(0) == S1->getOperand(0) &&
19154 S0->getOperand(1) == S1->getOperand(1)) {
19155 // Construct complete shuffle mask
19156 SmallVector<int, 8> Mask(S0->getMask());
19157 Mask.append(S1->getMask().begin(), S1->getMask().end());
19158
19159 if (isVMOVNTruncMask(Mask, VT, false))
19160 return DAG.getNode(
19161 ARMISD::VMOVN, DL, VT,
19162 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)),
19163 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)),
19164 DAG.getConstant(1, DL, MVT::i32));
19165 if (isVMOVNTruncMask(Mask, VT, true))
19166 return DAG.getNode(
19167 ARMISD::VMOVN, DL, VT,
19168 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)),
19169 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)),
19170 DAG.getConstant(1, DL, MVT::i32));
19171 }
19172 }
19173
19174 // For MVETrunc of a buildvector or shuffle, it can be beneficial to lower the
19175 // truncate to a buildvector to allow the generic optimisations to kick in.
19176 if (all_of(N->ops(), [](SDValue Op) {
19177 return Op.getOpcode() == ISD::BUILD_VECTOR ||
19178 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
19179 (Op.getOpcode() == ISD::BITCAST &&
19180 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
19181 })) {
19182 SmallVector<SDValue, 8> Extracts;
19183 for (unsigned Op = 0; Op < N->getNumOperands(); Op++) {
19184 SDValue O = N->getOperand(Op);
19185 for (unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
19186 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, O,
19187 DAG.getConstant(i, DL, MVT::i32));
19188 Extracts.push_back(Ext);
19189 }
19190 }
19191 return DAG.getBuildVector(VT, DL, Extracts);
19192 }
19193
19194 // If we are late in the legalization process and nothing has optimised
19195 // the trunc to anything better, lower it to a stack store and reload,
19196 // performing the truncation whilst keeping the lanes in the correct order:
19197 // VSTRH.32 a, stack; VSTRH.32 b, stack+8; VLDRW.32 stack;
19198 if (!DCI.isAfterLegalizeDAG())
19199 return SDValue();
19200
19201 SDValue StackPtr = DAG.CreateStackTemporary(TypeSize::getFixed(16), Align(4));
19202 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
19203 int NumIns = N->getNumOperands();
19204 assert((NumIns == 2 || NumIns == 4) &&
19205 "Expected 2 or 4 inputs to an MVETrunc");
19206 EVT StoreVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
19207 if (N->getNumOperands() == 4)
19208 StoreVT = StoreVT.getHalfNumVectorElementsVT(*DAG.getContext());
19209
19210 SmallVector<SDValue> Chains;
19211 for (int I = 0; I < NumIns; I++) {
19212 SDValue Ptr = DAG.getNode(
19213 ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
19214 DAG.getConstant(I * 16 / NumIns, DL, StackPtr.getValueType()));
19216 DAG.getMachineFunction(), SPFI, I * 16 / NumIns);
19217 SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), DL, N->getOperand(I),
19218 Ptr, MPI, StoreVT, Align(4));
19219 Chains.push_back(Ch);
19220 }
19221
19222 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
19223 MachinePointerInfo MPI =
19225 return DAG.getLoad(VT, DL, Chain, StackPtr, MPI, Align(4));
19226}
19227
19228// Take a MVEEXT(load x) and split that into (extload x, extload x+8)
19230 SelectionDAG &DAG) {
19231 SDValue N0 = N->getOperand(0);
19233 if (!LD || !LD->isSimple() || !N0.hasOneUse() || LD->isIndexed())
19234 return SDValue();
19235
19236 EVT FromVT = LD->getMemoryVT();
19237 EVT ToVT = N->getValueType(0);
19238 if (!ToVT.isVector())
19239 return SDValue();
19240 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements() * 2);
19241 EVT ToEltVT = ToVT.getVectorElementType();
19242 EVT FromEltVT = FromVT.getVectorElementType();
19243
19244 unsigned NumElements = 0;
19245 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
19246 NumElements = 4;
19247 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
19248 NumElements = 8;
19249 assert(NumElements != 0);
19250
19251 ISD::LoadExtType NewExtType =
19252 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
19253 if (LD->getExtensionType() != ISD::NON_EXTLOAD &&
19254 LD->getExtensionType() != ISD::EXTLOAD &&
19255 LD->getExtensionType() != NewExtType)
19256 return SDValue();
19257
19258 LLVMContext &C = *DAG.getContext();
19259 SDLoc DL(LD);
19260 // Details about the old load
19261 SDValue Ch = LD->getChain();
19262 SDValue BasePtr = LD->getBasePtr();
19263 Align Alignment = LD->getBaseAlign();
19264 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
19265 AAMDNodes AAInfo = LD->getAAInfo();
19266
19267 SDValue Offset = DAG.getPOISON(BasePtr.getValueType());
19268 EVT NewFromVT = EVT::getVectorVT(
19269 C, EVT::getIntegerVT(C, FromEltVT.getScalarSizeInBits()), NumElements);
19270 EVT NewToVT = EVT::getVectorVT(
19271 C, EVT::getIntegerVT(C, ToEltVT.getScalarSizeInBits()), NumElements);
19272
19275 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
19276 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
19277 SDValue NewPtr =
19278 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(NewOffset));
19279
19280 SDValue NewLoad =
19281 DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
19282 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
19283 Alignment, MMOFlags, AAInfo);
19284 Loads.push_back(NewLoad);
19285 Chains.push_back(SDValue(NewLoad.getNode(), 1));
19286 }
19287
19288 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
19289 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
19290 return DAG.getMergeValues(Loads, DL);
19291}
19292
19293// Perform combines for MVEEXT. If it has not be optimized to anything better
19294// before lowering, it gets converted to stack store and extloads performing the
19295// extend whilst still keeping the same lane ordering.
19298 SelectionDAG &DAG = DCI.DAG;
19299 EVT VT = N->getValueType(0);
19300 SDLoc DL(N);
19301 assert(N->getNumValues() == 2 && "Expected MVEEXT with 2 elements");
19302 assert((VT == MVT::v4i32 || VT == MVT::v8i16) && "Unexpected MVEEXT type");
19303
19304 EVT ExtVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
19305 *DAG.getContext());
19306 auto Extend = [&](SDValue V) {
19307 SDValue VVT = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, V);
19308 return N->getOpcode() == ARMISD::MVESEXT
19309 ? DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, VVT,
19310 DAG.getValueType(ExtVT))
19311 : DAG.getZeroExtendInReg(VVT, DL, ExtVT);
19312 };
19313
19314 // MVEEXT(VDUP) -> SIGN_EXTEND_INREG(VDUP)
19315 if (N->getOperand(0).getOpcode() == ARMISD::VDUP) {
19316 SDValue Ext = Extend(N->getOperand(0));
19317 return DAG.getMergeValues({Ext, Ext}, DL);
19318 }
19319
19320 // MVEEXT(shuffle) -> SIGN_EXTEND_INREG/ZERO_EXTEND_INREG
19321 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N->getOperand(0))) {
19322 ArrayRef<int> Mask = SVN->getMask();
19323 assert(Mask.size() == 2 * VT.getVectorNumElements());
19324 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
19325 unsigned Rev = VT == MVT::v4i32 ? ARMISD::VREV32 : ARMISD::VREV16;
19326 SDValue Op0 = SVN->getOperand(0);
19327 SDValue Op1 = SVN->getOperand(1);
19328
19329 auto CheckInregMask = [&](int Start, int Offset) {
19330 for (int Idx = 0, E = VT.getVectorNumElements(); Idx < E; ++Idx)
19331 if (Mask[Start + Idx] >= 0 && Mask[Start + Idx] != Idx * 2 + Offset)
19332 return false;
19333 return true;
19334 };
19335 SDValue V0 = SDValue(N, 0);
19336 SDValue V1 = SDValue(N, 1);
19337 if (CheckInregMask(0, 0))
19338 V0 = Extend(Op0);
19339 else if (CheckInregMask(0, 1))
19340 V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0));
19341 else if (CheckInregMask(0, Mask.size()))
19342 V0 = Extend(Op1);
19343 else if (CheckInregMask(0, Mask.size() + 1))
19344 V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1));
19345
19346 if (CheckInregMask(VT.getVectorNumElements(), Mask.size()))
19347 V1 = Extend(Op1);
19348 else if (CheckInregMask(VT.getVectorNumElements(), Mask.size() + 1))
19349 V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1));
19350 else if (CheckInregMask(VT.getVectorNumElements(), 0))
19351 V1 = Extend(Op0);
19352 else if (CheckInregMask(VT.getVectorNumElements(), 1))
19353 V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0));
19354
19355 if (V0.getNode() != N || V1.getNode() != N)
19356 return DAG.getMergeValues({V0, V1}, DL);
19357 }
19358
19359 // MVEEXT(load) -> extload, extload
19360 if (N->getOperand(0)->getOpcode() == ISD::LOAD)
19362 return L;
19363
19364 if (!DCI.isAfterLegalizeDAG())
19365 return SDValue();
19366
19367 // Lower to a stack store and reload:
19368 // VSTRW.32 a, stack; VLDRH.32 stack; VLDRH.32 stack+8;
19369 SDValue StackPtr = DAG.CreateStackTemporary(TypeSize::getFixed(16), Align(4));
19370 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
19371 int NumOuts = N->getNumValues();
19372 assert((NumOuts == 2 || NumOuts == 4) &&
19373 "Expected 2 or 4 outputs to an MVEEXT");
19374 EVT LoadVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
19375 *DAG.getContext());
19376 if (N->getNumOperands() == 4)
19377 LoadVT = LoadVT.getHalfNumVectorElementsVT(*DAG.getContext());
19378
19379 MachinePointerInfo MPI =
19381 SDValue Chain = DAG.getStore(DAG.getEntryNode(), DL, N->getOperand(0),
19382 StackPtr, MPI, Align(4));
19383
19385 for (int I = 0; I < NumOuts; I++) {
19386 SDValue Ptr = DAG.getNode(
19387 ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
19388 DAG.getConstant(I * 16 / NumOuts, DL, StackPtr.getValueType()));
19390 DAG.getMachineFunction(), SPFI, I * 16 / NumOuts);
19391 SDValue Load = DAG.getExtLoad(
19392 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD, DL,
19393 VT, Chain, Ptr, MPI, LoadVT, Align(4));
19394 Loads.push_back(Load);
19395 }
19396
19397 return DAG.getMergeValues(Loads, DL);
19398}
19399
19401 DAGCombinerInfo &DCI) const {
19402 switch (N->getOpcode()) {
19403 default: break;
19404 case ISD::SELECT_CC:
19405 case ISD::SELECT: return PerformSELECTCombine(N, DCI, Subtarget);
19406 case ISD::VSELECT: return PerformVSELECTCombine(N, DCI, Subtarget);
19407 case ISD::SETCC: return PerformVSetCCToVCTPCombine(N, DCI, Subtarget);
19408 case ARMISD::ADDE: return PerformADDECombine(N, DCI, Subtarget);
19409 case ARMISD::UMLAL: return PerformUMLALCombine(N, DCI.DAG, Subtarget);
19410 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget);
19411 case ISD::SUB: return PerformSUBCombine(N, DCI, Subtarget);
19412 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget);
19413 case ISD::OR: return PerformORCombine(N, DCI, Subtarget);
19414 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget);
19415 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget);
19416 case ISD::BRCOND:
19417 case ISD::BR_CC: return PerformHWLoopCombine(N, DCI, Subtarget);
19418 case ARMISD::ADDC:
19419 case ARMISD::SUBC: return PerformAddcSubcCombine(N, DCI, Subtarget);
19420 case ARMISD::SUBE: return PerformAddeSubeCombine(N, DCI, Subtarget);
19421 case ARMISD::BFI: return PerformBFICombine(N, DCI.DAG);
19422 case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
19423 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG);
19424 case ARMISD::VMOVhr: return PerformVMOVhrCombine(N, DCI);
19425 case ARMISD::VMOVrh: return PerformVMOVrhCombine(N, DCI.DAG);
19426 case ARMISD::VMOVrs:
19427 return PerformVMOVrsCombine(N, DCI.DAG);
19428 case ISD::STORE: return PerformSTORECombine(N, DCI, Subtarget);
19429 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
19432 return PerformExtractEltCombine(N, DCI, Subtarget);
19436 case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI, Subtarget);
19437 case ARMISD::VDUP: return PerformVDUPCombine(N, DCI.DAG, Subtarget);
19438 case ISD::FP_TO_SINT:
19439 case ISD::FP_TO_UINT:
19440 return PerformVCVTCombine(N, DCI.DAG, Subtarget);
19441 case ISD::FADD:
19442 return PerformFADDCombine(N, DCI.DAG, Subtarget);
19443 case ISD::FMUL:
19444 return PerformVMulVCTPCombine(N, DCI.DAG, Subtarget);
19446 return PerformIntrinsicCombine(N, DCI);
19447 case ISD::SHL:
19448 case ISD::SRA:
19449 case ISD::SRL:
19450 return PerformShiftCombine(N, DCI, Subtarget);
19451 case ISD::SIGN_EXTEND:
19452 case ISD::ZERO_EXTEND:
19453 case ISD::ANY_EXTEND:
19454 return PerformExtendCombine(N, DCI.DAG, Subtarget);
19455 case ISD::FP_EXTEND:
19456 return PerformFPExtendCombine(N, DCI.DAG, Subtarget);
19457 case ISD::SMIN:
19458 case ISD::UMIN:
19459 case ISD::SMAX:
19460 case ISD::UMAX:
19461 return PerformMinMaxCombine(N, DCI.DAG, Subtarget);
19462 case ARMISD::CMOV:
19463 return PerformCMOVCombine(N, DCI.DAG);
19464 case ARMISD::BRCOND:
19465 return PerformBRCONDCombine(N, DCI.DAG);
19466 case ARMISD::CMPZ:
19467 return PerformCMPZCombine(N, DCI.DAG);
19468 case ARMISD::CSINC:
19469 case ARMISD::CSINV:
19470 case ARMISD::CSNEG:
19471 return PerformCSETCombine(N, DCI.DAG);
19472 case ISD::LOAD:
19473 return PerformLOADCombine(N, DCI, Subtarget);
19475 return PerformVECTOR_DEINTERLEAVECombine(N, DCI, Subtarget);
19476 case ARMISD::VLD1DUP:
19477 case ARMISD::VLD2DUP:
19478 case ARMISD::VLD3DUP:
19479 case ARMISD::VLD4DUP:
19480 return PerformVLDCombine(N, DCI);
19482 return PerformARMBUILD_VECTORCombine(N, DCI);
19483 case ISD::BITCAST:
19484 return PerformBITCASTCombine(N, DCI, Subtarget);
19485 case ARMISD::PREDICATE_CAST:
19486 return PerformPREDICATE_CASTCombine(N, DCI);
19487 case ARMISD::VECTOR_REG_CAST:
19488 return PerformVECTOR_REG_CASTCombine(N, DCI.DAG, Subtarget);
19489 case ARMISD::MVETRUNC:
19490 return PerformMVETruncCombine(N, DCI);
19491 case ARMISD::MVESEXT:
19492 case ARMISD::MVEZEXT:
19493 return PerformMVEExtCombine(N, DCI);
19494 case ARMISD::VCMP:
19495 return PerformVCMPCombine(N, DCI.DAG, Subtarget);
19496 case ISD::VECREDUCE_ADD:
19497 return PerformVECREDUCE_ADDCombine(N, DCI.DAG, Subtarget);
19498 case ARMISD::VADDVs:
19499 case ARMISD::VADDVu:
19500 case ARMISD::VADDLVs:
19501 case ARMISD::VADDLVu:
19502 case ARMISD::VADDLVAs:
19503 case ARMISD::VADDLVAu:
19504 case ARMISD::VMLAVs:
19505 case ARMISD::VMLAVu:
19506 case ARMISD::VMLALVs:
19507 case ARMISD::VMLALVu:
19508 case ARMISD::VMLALVAs:
19509 case ARMISD::VMLALVAu:
19510 return PerformReduceShuffleCombine(N, DCI.DAG);
19511 case ARMISD::VMOVN:
19512 return PerformVMOVNCombine(N, DCI);
19513 case ARMISD::VQMOVNs:
19514 case ARMISD::VQMOVNu:
19515 return PerformVQMOVNCombine(N, DCI);
19516 case ARMISD::VQDMULH:
19517 return PerformVQDMULHCombine(N, DCI);
19518 case ARMISD::ASRL:
19519 case ARMISD::LSRL:
19520 case ARMISD::LSLL:
19521 return PerformLongShiftCombine(N, DCI.DAG);
19522 case ARMISD::SMULWB: {
19523 unsigned BitWidth = N->getValueType(0).getSizeInBits();
19524 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
19525 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
19526 return SDValue();
19527 break;
19528 }
19529 case ARMISD::SMULWT: {
19530 unsigned BitWidth = N->getValueType(0).getSizeInBits();
19531 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 16);
19532 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
19533 return SDValue();
19534 break;
19535 }
19536 case ARMISD::SMLALBB:
19537 case ARMISD::QADD16b:
19538 case ARMISD::QSUB16b:
19539 case ARMISD::UQADD16b:
19540 case ARMISD::UQSUB16b: {
19541 unsigned BitWidth = N->getValueType(0).getSizeInBits();
19542 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
19543 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) ||
19544 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)))
19545 return SDValue();
19546 break;
19547 }
19548 case ARMISD::SMLALBT: {
19549 unsigned LowWidth = N->getOperand(0).getValueType().getSizeInBits();
19550 APInt LowMask = APInt::getLowBitsSet(LowWidth, 16);
19551 unsigned HighWidth = N->getOperand(1).getValueType().getSizeInBits();
19552 APInt HighMask = APInt::getHighBitsSet(HighWidth, 16);
19553 if ((SimplifyDemandedBits(N->getOperand(0), LowMask, DCI)) ||
19554 (SimplifyDemandedBits(N->getOperand(1), HighMask, DCI)))
19555 return SDValue();
19556 break;
19557 }
19558 case ARMISD::SMLALTB: {
19559 unsigned HighWidth = N->getOperand(0).getValueType().getSizeInBits();
19560 APInt HighMask = APInt::getHighBitsSet(HighWidth, 16);
19561 unsigned LowWidth = N->getOperand(1).getValueType().getSizeInBits();
19562 APInt LowMask = APInt::getLowBitsSet(LowWidth, 16);
19563 if ((SimplifyDemandedBits(N->getOperand(0), HighMask, DCI)) ||
19564 (SimplifyDemandedBits(N->getOperand(1), LowMask, DCI)))
19565 return SDValue();
19566 break;
19567 }
19568 case ARMISD::SMLALTT: {
19569 unsigned BitWidth = N->getValueType(0).getSizeInBits();
19570 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 16);
19571 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) ||
19572 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)))
19573 return SDValue();
19574 break;
19575 }
19576 case ARMISD::QADD8b:
19577 case ARMISD::QSUB8b:
19578 case ARMISD::UQADD8b:
19579 case ARMISD::UQSUB8b: {
19580 unsigned BitWidth = N->getValueType(0).getSizeInBits();
19581 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 8);
19582 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) ||
19583 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)))
19584 return SDValue();
19585 break;
19586 }
19587 case ARMISD::VBSP:
19588 if (N->getOperand(1) == N->getOperand(2))
19589 return N->getOperand(1);
19590 return SDValue();
19593 switch (N->getConstantOperandVal(1)) {
19594 case Intrinsic::arm_neon_vld1:
19595 case Intrinsic::arm_neon_vld1x2:
19596 case Intrinsic::arm_neon_vld1x3:
19597 case Intrinsic::arm_neon_vld1x4:
19598 case Intrinsic::arm_neon_vld2:
19599 case Intrinsic::arm_neon_vld3:
19600 case Intrinsic::arm_neon_vld4:
19601 case Intrinsic::arm_neon_vld2lane:
19602 case Intrinsic::arm_neon_vld3lane:
19603 case Intrinsic::arm_neon_vld4lane:
19604 case Intrinsic::arm_neon_vld2dup:
19605 case Intrinsic::arm_neon_vld3dup:
19606 case Intrinsic::arm_neon_vld4dup:
19607 case Intrinsic::arm_neon_vst1:
19608 case Intrinsic::arm_neon_vst1x2:
19609 case Intrinsic::arm_neon_vst1x3:
19610 case Intrinsic::arm_neon_vst1x4:
19611 case Intrinsic::arm_neon_vst2:
19612 case Intrinsic::arm_neon_vst3:
19613 case Intrinsic::arm_neon_vst4:
19614 case Intrinsic::arm_neon_vst2lane:
19615 case Intrinsic::arm_neon_vst3lane:
19616 case Intrinsic::arm_neon_vst4lane:
19617 return PerformVLDCombine(N, DCI);
19618 case Intrinsic::arm_mve_vld2q:
19619 case Intrinsic::arm_mve_vld4q:
19620 case Intrinsic::arm_mve_vst2q:
19621 case Intrinsic::arm_mve_vst4q:
19622 return PerformMVEVLDCombine(N, DCI);
19623 default: break;
19624 }
19625 break;
19626 }
19627 return SDValue();
19628}
19629
19631 EVT VT) const {
19632 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
19633}
19634
19636 Align Alignment,
19638 unsigned *Fast) const {
19639 // Depends what it gets converted into if the type is weird.
19640 if (!VT.isSimple())
19641 return false;
19642
19643 // The AllowsUnaligned flag models the SCTLR.A setting in ARM cpus
19644 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
19645 auto Ty = VT.getSimpleVT().SimpleTy;
19646
19647 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19648 // Unaligned access can use (for example) LRDB, LRDH, LDR
19649 if (AllowsUnaligned) {
19650 if (Fast)
19651 *Fast = Subtarget->hasV7Ops();
19652 return true;
19653 }
19654 }
19655
19656 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19657 // For any little-endian targets with neon, we can support unaligned ld/st
19658 // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
19659 // A big-endian target may also explicitly support unaligned accesses
19660 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
19661 if (Fast)
19662 *Fast = 1;
19663 return true;
19664 }
19665 }
19666
19667 if (!Subtarget->hasMVEIntegerOps())
19668 return false;
19669
19670 // These are for predicates
19671 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19672 Ty == MVT::v2i1)) {
19673 if (Fast)
19674 *Fast = 1;
19675 return true;
19676 }
19677
19678 // These are for truncated stores/narrowing loads. They are fine so long as
19679 // the alignment is at least the size of the item being loaded
19680 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19681 Alignment >= VT.getScalarSizeInBits() / 8) {
19682 if (Fast)
19683 *Fast = true;
19684 return true;
19685 }
19686
19687 // In little-endian MVE, the store instructions VSTRB.U8, VSTRH.U16 and
19688 // VSTRW.U32 all store the vector register in exactly the same format, and
19689 // differ only in the range of their immediate offset field and the required
19690 // alignment. So there is always a store that can be used, regardless of
19691 // actual type.
19692 //
19693 // For big endian, that is not the case. But can still emit a (VSTRB.U8;
19694 // VREV64.8) pair and get the same effect. This will likely be better than
19695 // aligning the vector through the stack.
19696 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19697 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19698 Ty == MVT::v2f64) {
19699 if (Fast)
19700 *Fast = 1;
19701 return true;
19702 }
19703
19704 return false;
19705}
19706
19708 LLVMContext &Context, const MemOp &Op,
19709 const AttributeList &FuncAttributes) const {
19710 // See if we can use NEON instructions for this...
19711 if ((Op.isMemcpyOrMemmove() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
19712 !FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {
19713 unsigned Fast;
19714 if (Op.size() >= 16 &&
19715 (Op.isAligned(Align(16)) ||
19716 (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, Align(1),
19718 Fast))) {
19719 return MVT::v2f64;
19720 } else if (Op.size() >= 8 &&
19721 (Op.isAligned(Align(8)) ||
19723 MVT::f64, 0, Align(1), MachineMemOperand::MONone, &Fast) &&
19724 Fast))) {
19725 return MVT::f64;
19726 }
19727 }
19728
19729 // Let the target-independent logic figure it out.
19730 return MVT::Other;
19731}
19732
19733// 64-bit integers are split into their high and low parts and held in two
19734// different registers, so the trunc is free since the low register can just
19735// be used.
19736bool ARMTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
19737 if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
19738 return false;
19739 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
19740 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
19741 return (SrcBits == 64 && DestBits == 32);
19742}
19743
19745 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
19746 !DstVT.isInteger())
19747 return false;
19748 unsigned SrcBits = SrcVT.getSizeInBits();
19749 unsigned DestBits = DstVT.getSizeInBits();
19750 return (SrcBits == 64 && DestBits == 32);
19751}
19752
19754 if (Val.getOpcode() != ISD::LOAD)
19755 return false;
19756
19757 EVT VT1 = Val.getValueType();
19758 if (!VT1.isSimple() || !VT1.isInteger() ||
19759 !VT2.isSimple() || !VT2.isInteger())
19760 return false;
19761
19762 switch (VT1.getSimpleVT().SimpleTy) {
19763 default: break;
19764 case MVT::i1:
19765 case MVT::i8:
19766 case MVT::i16:
19767 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits.
19768 return true;
19769 }
19770
19771 return false;
19772}
19773
19775 if (!VT.isSimple())
19776 return false;
19777
19778 // There are quite a few FP16 instructions (e.g. VNMLA, VNMLS, etc.) that
19779 // negate values directly (fneg is free). So, we don't want to let the DAG
19780 // combiner rewrite fneg into xors and some other instructions. For f16 and
19781 // FullFP16 argument passing, some bitcast nodes may be introduced,
19782 // triggering this DAG combine rewrite, so we are avoiding that with this.
19783 switch (VT.getSimpleVT().SimpleTy) {
19784 default: break;
19785 case MVT::f16:
19786 return Subtarget->hasFullFP16();
19787 }
19788
19789 return false;
19790}
19791
19793 if (!Subtarget->hasMVEIntegerOps())
19794 return nullptr;
19795 Type *SVIType = SVI->getType();
19796 Type *ScalarType = SVIType->getScalarType();
19797
19798 if (ScalarType->isFloatTy())
19799 return Type::getInt32Ty(SVIType->getContext());
19800 if (ScalarType->isHalfTy())
19801 return Type::getInt16Ty(SVIType->getContext());
19802 return nullptr;
19803}
19804
19806 EVT VT = ExtVal.getValueType();
19807
19808 if (!isTypeLegal(VT))
19809 return false;
19810
19811 if (auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.getOperand(0))) {
19812 if (Ld->isExpandingLoad())
19813 return false;
19814 }
19815
19816 if (Subtarget->hasMVEIntegerOps())
19817 return true;
19818
19819 // Don't create a loadext if we can fold the extension into a wide/long
19820 // instruction.
19821 // If there's more than one user instruction, the loadext is desirable no
19822 // matter what. There can be two uses by the same instruction.
19823 if (ExtVal->use_empty() ||
19824 !ExtVal->user_begin()->isOnlyUserOf(ExtVal.getNode()))
19825 return true;
19826
19827 SDNode *U = *ExtVal->user_begin();
19828 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB ||
19829 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHLIMM))
19830 return false;
19831
19832 return true;
19833}
19834
19836 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
19837 return false;
19838
19839 if (!isTypeLegal(EVT::getEVT(Ty1)))
19840 return false;
19841
19842 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
19843
19844 // Assuming the caller doesn't have a zeroext or signext return parameter,
19845 // truncation all the way down to i1 is valid.
19846 return true;
19847}
19848
19849/// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
19850/// than a pair of fmul and fadd instructions. fmuladd intrinsics will be
19851/// expanded to FMAs when this method returns true, otherwise fmuladd is
19852/// expanded to fmul + fadd.
19853///
19854/// ARM supports both fused and unfused multiply-add operations; we already
19855/// lower a pair of fmul and fadd to the latter so it's not clear that there
19856/// would be a gain or that the gain would be worthwhile enough to risk
19857/// correctness bugs.
19858///
19859/// For MVE, we set this to true as it helps simplify the need for some
19860/// patterns (and we don't have the non-fused floating point instruction).
19861bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
19862 EVT VT) const {
19863 if (Subtarget->useSoftFloat())
19864 return false;
19865
19866 if (!VT.isSimple())
19867 return false;
19868
19869 switch (VT.getSimpleVT().SimpleTy) {
19870 case MVT::v4f32:
19871 case MVT::v8f16:
19872 return Subtarget->hasMVEFloatOps();
19873 case MVT::f16:
19874 return Subtarget->useFPVFMx16();
19875 case MVT::f32:
19876 return Subtarget->useFPVFMx();
19877 case MVT::f64:
19878 return Subtarget->useFPVFMx64();
19879 default:
19880 break;
19881 }
19882
19883 return false;
19884}
19885
19886static bool isLegalT1AddressImmediate(int64_t V, EVT VT) {
19887 if (V < 0)
19888 return false;
19889
19890 unsigned Scale = 1;
19891 switch (VT.getSimpleVT().SimpleTy) {
19892 case MVT::i1:
19893 case MVT::i8:
19894 // Scale == 1;
19895 break;
19896 case MVT::i16:
19897 // Scale == 2;
19898 Scale = 2;
19899 break;
19900 default:
19901 // On thumb1 we load most things (i32, i64, floats, etc) with a LDR
19902 // Scale == 4;
19903 Scale = 4;
19904 break;
19905 }
19906
19907 if ((V & (Scale - 1)) != 0)
19908 return false;
19909 return isUInt<5>(V / Scale);
19910}
19911
19912static bool isLegalT2AddressImmediate(int64_t V, EVT VT,
19913 const ARMSubtarget *Subtarget) {
19914 if (!VT.isInteger() && !VT.isFloatingPoint())
19915 return false;
19916 if (VT.isVector() && Subtarget->hasNEON())
19917 return false;
19918 if (VT.isVector() && VT.isFloatingPoint() && Subtarget->hasMVEIntegerOps() &&
19919 !Subtarget->hasMVEFloatOps())
19920 return false;
19921
19922 bool IsNeg = false;
19923 if (V < 0) {
19924 IsNeg = true;
19925 V = -V;
19926 }
19927
19928 unsigned NumBytes = std::max((unsigned)VT.getSizeInBits() / 8, 1U);
19929
19930 // MVE: size * imm7
19931 if (VT.isVector() && Subtarget->hasMVEIntegerOps()) {
19932 switch (VT.getSimpleVT().getVectorElementType().SimpleTy) {
19933 case MVT::i32:
19934 case MVT::f32:
19935 return isShiftedUInt<7,2>(V);
19936 case MVT::i16:
19937 case MVT::f16:
19938 return isShiftedUInt<7,1>(V);
19939 case MVT::i8:
19940 return isUInt<7>(V);
19941 default:
19942 return false;
19943 }
19944 }
19945
19946 // half VLDR: 2 * imm8
19947 if (VT.isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19948 return isShiftedUInt<8, 1>(V);
19949 // VLDR and LDRD: 4 * imm8
19950 if ((VT.isFloatingPoint() && Subtarget->hasVFP2Base()) || NumBytes == 8)
19951 return isShiftedUInt<8, 2>(V);
19952
19953 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19954 // + imm12 or - imm8
19955 if (IsNeg)
19956 return isUInt<8>(V);
19957 return isUInt<12>(V);
19958 }
19959
19960 return false;
19961}
19962
19963/// isLegalAddressImmediate - Return true if the integer value can be used
19964/// as the offset of the target addressing mode for load / store of the
19965/// given type.
19966static bool isLegalAddressImmediate(int64_t V, EVT VT,
19967 const ARMSubtarget *Subtarget) {
19968 if (V == 0)
19969 return true;
19970
19971 if (!VT.isSimple())
19972 return false;
19973
19974 if (Subtarget->isThumb1Only())
19975 return isLegalT1AddressImmediate(V, VT);
19976 else if (Subtarget->isThumb2())
19978
19979 // ARM mode.
19980 if (V < 0)
19981 V = - V;
19982 switch (VT.getSimpleVT().SimpleTy) {
19983 default: return false;
19984 case MVT::i1:
19985 case MVT::i8:
19986 case MVT::i32:
19987 // +- imm12
19988 return isUInt<12>(V);
19989 case MVT::i16:
19990 // +- imm8
19991 return isUInt<8>(V);
19992 case MVT::f32:
19993 case MVT::f64:
19994 if (!Subtarget->hasVFP2Base()) // FIXME: NEON?
19995 return false;
19996 return isShiftedUInt<8, 2>(V);
19997 }
19998}
19999
20001 EVT VT) const {
20002 int Scale = AM.Scale;
20003 if (Scale < 0)
20004 return false;
20005
20006 switch (VT.getSimpleVT().SimpleTy) {
20007 default: return false;
20008 case MVT::i1:
20009 case MVT::i8:
20010 case MVT::i16:
20011 case MVT::i32:
20012 if (Scale == 1)
20013 return true;
20014 // r + r << imm
20015 Scale = Scale & ~1;
20016 return Scale == 2 || Scale == 4 || Scale == 8;
20017 case MVT::i64:
20018 // FIXME: What are we trying to model here? ldrd doesn't have an r + r
20019 // version in Thumb mode.
20020 // r + r
20021 if (Scale == 1)
20022 return true;
20023 // r * 2 (this can be lowered to r + r).
20024 if (!AM.HasBaseReg && Scale == 2)
20025 return true;
20026 return false;
20027 case MVT::isVoid:
20028 // Note, we allow "void" uses (basically, uses that aren't loads or
20029 // stores), because arm allows folding a scale into many arithmetic
20030 // operations. This should be made more precise and revisited later.
20031
20032 // Allow r << imm, but the imm has to be a multiple of two.
20033 if (Scale & 1) return false;
20034 return isPowerOf2_32(Scale);
20035 }
20036}
20037
20039 EVT VT) const {
20040 const int Scale = AM.Scale;
20041
20042 // Negative scales are not supported in Thumb1.
20043 if (Scale < 0)
20044 return false;
20045
20046 // Thumb1 addressing modes do not support register scaling excepting the
20047 // following cases:
20048 // 1. Scale == 1 means no scaling.
20049 // 2. Scale == 2 this can be lowered to r + r if there is no base register.
20050 return (Scale == 1) || (!AM.HasBaseReg && Scale == 2);
20051}
20052
20053/// isLegalAddressingMode - Return true if the addressing mode represented
20054/// by AM is legal for this target, for a load/store of the specified type.
20056 const AddrMode &AM, Type *Ty,
20057 unsigned AS, Instruction *I) const {
20058 EVT VT = getValueType(DL, Ty, true);
20059 if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget))
20060 return false;
20061
20062 // Can never fold addr of global into load/store.
20063 if (AM.BaseGV)
20064 return false;
20065
20066 switch (AM.Scale) {
20067 case 0: // no scale reg, must be "r+i" or "r", or "i".
20068 break;
20069 default:
20070 // ARM doesn't support any R+R*scale+imm addr modes.
20071 if (AM.BaseOffs)
20072 return false;
20073
20074 if (!VT.isSimple())
20075 return false;
20076
20077 if (Subtarget->isThumb1Only())
20078 return isLegalT1ScaledAddressingMode(AM, VT);
20079
20080 if (Subtarget->isThumb2())
20081 return isLegalT2ScaledAddressingMode(AM, VT);
20082
20083 int Scale = AM.Scale;
20084 switch (VT.getSimpleVT().SimpleTy) {
20085 default: return false;
20086 case MVT::i1:
20087 case MVT::i8:
20088 case MVT::i32:
20089 if (Scale < 0) Scale = -Scale;
20090 if (Scale == 1)
20091 return true;
20092 // r + r << imm
20093 return isPowerOf2_32(Scale & ~1);
20094 case MVT::i16:
20095 case MVT::i64:
20096 // r +/- r
20097 if (Scale == 1 || (AM.HasBaseReg && Scale == -1))
20098 return true;
20099 // r * 2 (this can be lowered to r + r).
20100 if (!AM.HasBaseReg && Scale == 2)
20101 return true;
20102 return false;
20103
20104 case MVT::isVoid:
20105 // Note, we allow "void" uses (basically, uses that aren't loads or
20106 // stores), because arm allows folding a scale into many arithmetic
20107 // operations. This should be made more precise and revisited later.
20108
20109 // Allow r << imm, but the imm has to be a multiple of two.
20110 if (Scale & 1) return false;
20111 return isPowerOf2_32(Scale);
20112 }
20113 }
20114 return true;
20115}
20116
20117/// isLegalICmpImmediate - Return true if the specified immediate is legal
20118/// icmp immediate, that is the target has icmp instructions which can compare
20119/// a register against the immediate without having to materialize the
20120/// immediate into a register.
20122 // Thumb2 and ARM modes can use cmn for negative immediates.
20123 if (!Subtarget->isThumb())
20124 return ARM_AM::getSOImmVal((uint32_t)Imm) != -1 ||
20126 if (Subtarget->isThumb2())
20127 return ARM_AM::getT2SOImmVal((uint32_t)Imm) != -1 ||
20129 // Thumb1 doesn't have cmn, and only 8-bit immediates.
20130 return Imm >= 0 && Imm <= 255;
20131}
20132
20133/// isLegalAddImmediate - Return true if the specified immediate is a legal add
20134/// *or sub* immediate, that is the target has add or sub instructions which can
20135/// add a register with the immediate without having to materialize the
20136/// immediate into a register.
20138 // Same encoding for add/sub, just flip the sign.
20139 uint64_t AbsImm = AbsoluteValue(Imm);
20140 if (!Subtarget->isThumb())
20141 return ARM_AM::getSOImmVal(AbsImm) != -1;
20142 if (Subtarget->isThumb2())
20143 return ARM_AM::getT2SOImmVal(AbsImm) != -1;
20144 // Thumb1 only has 8-bit unsigned immediate.
20145 return AbsImm <= 255;
20146}
20147
20148// Return false to prevent folding
20149// (mul (add r, c0), c1) -> (add (mul r, c1), c0*c1) in DAGCombine,
20150// if the folding leads to worse code.
20152 SDValue ConstNode) const {
20153 // Let the DAGCombiner decide for vector types and large types.
20154 const EVT VT = AddNode.getValueType();
20155 if (VT.isVector() || VT.getScalarSizeInBits() > 32)
20156 return true;
20157
20158 // It is worse if c0 is legal add immediate, while c1*c0 is not
20159 // and has to be composed by at least two instructions.
20160 const ConstantSDNode *C0Node = cast<ConstantSDNode>(AddNode.getOperand(1));
20161 const ConstantSDNode *C1Node = cast<ConstantSDNode>(ConstNode);
20162 const int64_t C0 = C0Node->getSExtValue();
20163 APInt CA = C0Node->getAPIntValue() * C1Node->getAPIntValue();
20165 return true;
20166 if (ConstantMaterializationCost((unsigned)CA.getZExtValue(), Subtarget) > 1)
20167 return false;
20168
20169 // Default to true and let the DAGCombiner decide.
20170 return true;
20171}
20172
20174 bool isSEXTLoad, SDValue &Base,
20175 SDValue &Offset, bool &isInc,
20176 SelectionDAG &DAG) {
20177 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
20178 return false;
20179
20180 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
20181 // AddressingMode 3
20182 Base = Ptr->getOperand(0);
20184 int RHSC = (int)RHS->getZExtValue();
20185 if (RHSC < 0 && RHSC > -256) {
20186 assert(Ptr->getOpcode() == ISD::ADD);
20187 isInc = false;
20188 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
20189 return true;
20190 }
20191 }
20192 isInc = (Ptr->getOpcode() == ISD::ADD);
20193 Offset = Ptr->getOperand(1);
20194 return true;
20195 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
20196 // AddressingMode 2
20198 int RHSC = (int)RHS->getZExtValue();
20199 if (RHSC < 0 && RHSC > -0x1000) {
20200 assert(Ptr->getOpcode() == ISD::ADD);
20201 isInc = false;
20202 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
20203 Base = Ptr->getOperand(0);
20204 return true;
20205 }
20206 }
20207
20208 if (Ptr->getOpcode() == ISD::ADD) {
20209 isInc = true;
20210 ARM_AM::ShiftOpc ShOpcVal=
20212 if (ShOpcVal != ARM_AM::no_shift) {
20213 Base = Ptr->getOperand(1);
20214 Offset = Ptr->getOperand(0);
20215 } else {
20216 Base = Ptr->getOperand(0);
20217 Offset = Ptr->getOperand(1);
20218 }
20219 return true;
20220 }
20221
20222 isInc = (Ptr->getOpcode() == ISD::ADD);
20223 Base = Ptr->getOperand(0);
20224 Offset = Ptr->getOperand(1);
20225 return true;
20226 }
20227
20228 // FIXME: Use VLDM / VSTM to emulate indexed FP load / store.
20229 return false;
20230}
20231
20233 bool isSEXTLoad, SDValue &Base,
20234 SDValue &Offset, bool &isInc,
20235 SelectionDAG &DAG) {
20236 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
20237 return false;
20238
20239 Base = Ptr->getOperand(0);
20241 int RHSC = (int)RHS->getZExtValue();
20242 if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
20243 assert(Ptr->getOpcode() == ISD::ADD);
20244 isInc = false;
20245 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
20246 return true;
20247 } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
20248 isInc = Ptr->getOpcode() == ISD::ADD;
20249 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
20250 return true;
20251 }
20252 }
20253
20254 return false;
20255}
20256
20257static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment,
20258 bool isSEXTLoad, bool IsMasked, bool isLE,
20260 bool &isInc, SelectionDAG &DAG) {
20261 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
20262 return false;
20263 if (!isa<ConstantSDNode>(Ptr->getOperand(1)))
20264 return false;
20265
20266 // We allow LE non-masked loads to change the type (for example use a vldrb.8
20267 // as opposed to a vldrw.32). This can allow extra addressing modes or
20268 // alignments for what is otherwise an equivalent instruction.
20269 bool CanChangeType = isLE && !IsMasked;
20270
20272 int RHSC = (int)RHS->getZExtValue();
20273
20274 auto IsInRange = [&](int RHSC, int Limit, int Scale) {
20275 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
20276 assert(Ptr->getOpcode() == ISD::ADD);
20277 isInc = false;
20278 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
20279 return true;
20280 } else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
20281 isInc = Ptr->getOpcode() == ISD::ADD;
20282 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
20283 return true;
20284 }
20285 return false;
20286 };
20287
20288 // Try to find a matching instruction based on s/zext, Alignment, Offset and
20289 // (in BE/masked) type.
20290 Base = Ptr->getOperand(0);
20291 if (VT == MVT::v4i16) {
20292 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
20293 return true;
20294 } else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
20295 if (IsInRange(RHSC, 0x80, 1))
20296 return true;
20297 } else if (Alignment >= 4 &&
20298 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
20299 IsInRange(RHSC, 0x80, 4))
20300 return true;
20301 else if (Alignment >= 2 &&
20302 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
20303 IsInRange(RHSC, 0x80, 2))
20304 return true;
20305 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
20306 return true;
20307 return false;
20308}
20309
20310/// getPreIndexedAddressParts - returns true by value, base pointer and
20311/// offset pointer and addressing mode by reference if the node's address
20312/// can be legally represented as pre-indexed load / store address.
20313bool
20315 SDValue &Offset,
20317 SelectionDAG &DAG) const {
20318 if (Subtarget->isThumb1Only())
20319 return false;
20320
20321 EVT VT;
20322 SDValue Ptr;
20323 Align Alignment;
20324 unsigned AS = 0;
20325 bool isSEXTLoad = false;
20326 bool IsMasked = false;
20327 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
20328 Ptr = LD->getBasePtr();
20329 VT = LD->getMemoryVT();
20330 Alignment = LD->getAlign();
20331 AS = LD->getAddressSpace();
20332 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20333 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
20334 Ptr = ST->getBasePtr();
20335 VT = ST->getMemoryVT();
20336 Alignment = ST->getAlign();
20337 AS = ST->getAddressSpace();
20338 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) {
20339 Ptr = LD->getBasePtr();
20340 VT = LD->getMemoryVT();
20341 Alignment = LD->getAlign();
20342 AS = LD->getAddressSpace();
20343 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20344 IsMasked = true;
20346 Ptr = ST->getBasePtr();
20347 VT = ST->getMemoryVT();
20348 Alignment = ST->getAlign();
20349 AS = ST->getAddressSpace();
20350 IsMasked = true;
20351 } else
20352 return false;
20353
20354 unsigned Fast = 0;
20355 if (!allowsMisalignedMemoryAccesses(VT, AS, Alignment,
20357 // Only generate post-increment or pre-increment forms when a real
20358 // hardware instruction exists for them. Do not emit postinc/preinc
20359 // if the operation will end up as a libcall.
20360 return false;
20361 }
20362
20363 bool isInc;
20364 bool isLegal = false;
20365 if (VT.isVector())
20366 isLegal = Subtarget->hasMVEIntegerOps() &&
20368 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
20369 Subtarget->isLittle(), Base, Offset, isInc, DAG);
20370 else {
20371 if (Subtarget->isThumb2())
20372 isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
20373 Offset, isInc, DAG);
20374 else
20375 isLegal = getARMIndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
20376 Offset, isInc, DAG);
20377 }
20378 if (!isLegal)
20379 return false;
20380
20381 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC;
20382 return true;
20383}
20384
20385/// getPostIndexedAddressParts - returns true by value, base pointer and
20386/// offset pointer and addressing mode by reference if this node can be
20387/// combined with a load / store to form a post-indexed load / store.
20389 SDValue &Base,
20390 SDValue &Offset,
20392 SelectionDAG &DAG) const {
20393 EVT VT;
20394 SDValue Ptr;
20395 Align Alignment;
20396 bool isSEXTLoad = false, isNonExt;
20397 bool IsMasked = false;
20398 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
20399 VT = LD->getMemoryVT();
20400 Ptr = LD->getBasePtr();
20401 Alignment = LD->getAlign();
20402 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20403 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
20404 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
20405 VT = ST->getMemoryVT();
20406 Ptr = ST->getBasePtr();
20407 Alignment = ST->getAlign();
20408 isNonExt = !ST->isTruncatingStore();
20409 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) {
20410 VT = LD->getMemoryVT();
20411 Ptr = LD->getBasePtr();
20412 Alignment = LD->getAlign();
20413 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20414 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
20415 IsMasked = true;
20417 VT = ST->getMemoryVT();
20418 Ptr = ST->getBasePtr();
20419 Alignment = ST->getAlign();
20420 isNonExt = !ST->isTruncatingStore();
20421 IsMasked = true;
20422 } else
20423 return false;
20424
20425 if (Subtarget->isThumb1Only()) {
20426 // Thumb-1 can do a limited post-inc load or store as an updating LDM. It
20427 // must be non-extending/truncating, i32, with an offset of 4.
20428 assert(Op->getValueType(0) == MVT::i32 && "Non-i32 post-inc op?!");
20429 if (Op->getOpcode() != ISD::ADD || !isNonExt)
20430 return false;
20431 auto *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1));
20432 if (!RHS || RHS->getZExtValue() != 4)
20433 return false;
20434 if (Alignment < Align(4))
20435 return false;
20436
20437 Offset = Op->getOperand(1);
20438 Base = Op->getOperand(0);
20439 AM = ISD::POST_INC;
20440 return true;
20441 }
20442
20443 bool isInc;
20444 bool isLegal = false;
20445 if (VT.isVector())
20446 isLegal = Subtarget->hasMVEIntegerOps() &&
20447 getMVEIndexedAddressParts(Op, VT, Alignment, isSEXTLoad, IsMasked,
20448 Subtarget->isLittle(), Base, Offset,
20449 isInc, DAG);
20450 else {
20451 if (Subtarget->isThumb2())
20452 isLegal = getT2IndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
20453 isInc, DAG);
20454 else
20455 isLegal = getARMIndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
20456 isInc, DAG);
20457 }
20458 if (!isLegal)
20459 return false;
20460
20461 if (Ptr != Base) {
20462 // Swap base ptr and offset to catch more post-index load / store when
20463 // it's legal. In Thumb2 mode, offset must be an immediate.
20464 if (Ptr == Offset && Op->getOpcode() == ISD::ADD &&
20465 !Subtarget->isThumb2())
20467
20468 // Post-indexed load / store update the base pointer.
20469 if (Ptr != Base)
20470 return false;
20471 }
20472
20473 AM = isInc ? ISD::POST_INC : ISD::POST_DEC;
20474 return true;
20475}
20476
20479 const APInt &DemandedElts,
20480 const SelectionDAG &DAG,
20481 unsigned Depth) const {
20482 unsigned BitWidth = Known.getBitWidth();
20483 Known.resetAll();
20484 switch (Op.getOpcode()) {
20485 default: break;
20486 case ARMISD::ADDC:
20487 case ARMISD::ADDE:
20488 case ARMISD::SUBC:
20489 case ARMISD::SUBE:
20490 // Special cases when we convert a carry to a boolean.
20491 if (Op.getResNo() == 0) {
20492 SDValue LHS = Op.getOperand(0);
20493 SDValue RHS = Op.getOperand(1);
20494 // (ADDE 0, 0, C) will give us a single bit.
20495 if (Op->getOpcode() == ARMISD::ADDE && isNullConstant(LHS) &&
20496 isNullConstant(RHS)) {
20498 return;
20499 }
20500 }
20501 break;
20502 case ARMISD::CMOV: {
20503 // Bits are known zero/one if known on the LHS and RHS.
20504 Known = DAG.computeKnownBits(Op.getOperand(0), Depth+1);
20505 if (Known.isUnknown())
20506 return;
20507
20508 KnownBits KnownRHS = DAG.computeKnownBits(Op.getOperand(1), Depth+1);
20509 Known = Known.intersectWith(KnownRHS);
20510 return;
20511 }
20513 Intrinsic::ID IntID =
20514 static_cast<Intrinsic::ID>(Op->getConstantOperandVal(1));
20515 switch (IntID) {
20516 default: return;
20517 case Intrinsic::arm_ldaex:
20518 case Intrinsic::arm_ldrex: {
20519 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
20520 unsigned MemBits = VT.getScalarSizeInBits();
20521 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
20522 return;
20523 }
20524 }
20525 }
20526 case ARMISD::BFI: {
20527 // Conservatively, we can recurse down the first operand
20528 // and just mask out all affected bits.
20529 Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
20530
20531 // The operand to BFI is already a mask suitable for removing the bits it
20532 // sets.
20533 const APInt &Mask = Op.getConstantOperandAPInt(2);
20534 Known.Zero &= Mask;
20535 Known.One &= Mask;
20536 return;
20537 }
20538 case ARMISD::VGETLANEs:
20539 case ARMISD::VGETLANEu: {
20540 const SDValue &SrcSV = Op.getOperand(0);
20541 EVT VecVT = SrcSV.getValueType();
20542 assert(VecVT.isVector() && "VGETLANE expected a vector type");
20543 const unsigned NumSrcElts = VecVT.getVectorNumElements();
20544 ConstantSDNode *Pos = cast<ConstantSDNode>(Op.getOperand(1).getNode());
20545 assert(Pos->getAPIntValue().ult(NumSrcElts) &&
20546 "VGETLANE index out of bounds");
20547 unsigned Idx = Pos->getZExtValue();
20548 APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
20549 Known = DAG.computeKnownBits(SrcSV, DemandedElt, Depth + 1);
20550
20551 EVT VT = Op.getValueType();
20552 const unsigned DstSz = VT.getScalarSizeInBits();
20553 const unsigned SrcSz = VecVT.getVectorElementType().getSizeInBits();
20554 (void)SrcSz;
20555 assert(SrcSz == Known.getBitWidth());
20556 assert(DstSz > SrcSz);
20557 if (Op.getOpcode() == ARMISD::VGETLANEs)
20558 Known = Known.sext(DstSz);
20559 else {
20560 Known = Known.zext(DstSz);
20561 }
20562 assert(DstSz == Known.getBitWidth());
20563 break;
20564 }
20565 case ARMISD::VMOVrh: {
20566 KnownBits KnownOp = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
20567 assert(KnownOp.getBitWidth() == 16);
20568 Known = KnownOp.zext(32);
20569 break;
20570 }
20571 case ARMISD::CSINC:
20572 case ARMISD::CSINV:
20573 case ARMISD::CSNEG: {
20574 KnownBits KnownOp0 = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
20575 KnownBits KnownOp1 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
20576
20577 // The result is either:
20578 // CSINC: KnownOp0 or KnownOp1 + 1
20579 // CSINV: KnownOp0 or ~KnownOp1
20580 // CSNEG: KnownOp0 or KnownOp1 * -1
20581 if (Op.getOpcode() == ARMISD::CSINC)
20582 KnownOp1 =
20583 KnownBits::add(KnownOp1, KnownBits::makeConstant(APInt(32, 1)));
20584 else if (Op.getOpcode() == ARMISD::CSINV)
20585 std::swap(KnownOp1.Zero, KnownOp1.One);
20586 else if (Op.getOpcode() == ARMISD::CSNEG)
20587 KnownOp1 = KnownBits::mul(KnownOp1,
20589
20590 Known = KnownOp0.intersectWith(KnownOp1);
20591 break;
20592 }
20593 case ARMISD::VORRIMM:
20594 case ARMISD::VBICIMM: {
20595 unsigned Encoded = Op.getConstantOperandVal(1);
20596 unsigned DecEltBits = 0;
20597 uint64_t DecodedVal = ARM_AM::decodeVMOVModImm(Encoded, DecEltBits);
20598
20599 unsigned EltBits = Op.getScalarValueSizeInBits();
20600 if (EltBits != DecEltBits) {
20601 // Be conservative: only update Known when EltBits == DecEltBits.
20602 // This is believed to always be true for VORRIMM/VBICIMM today, but if
20603 // that changes in the future, doing nothing here is safer than risking
20604 // subtle bugs.
20605 break;
20606 }
20607
20608 KnownBits KnownLHS = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
20609 bool IsVORR = Op.getOpcode() == ARMISD::VORRIMM;
20610 APInt Imm(DecEltBits, DecodedVal);
20611
20612 Known.One = IsVORR ? (KnownLHS.One | Imm) : (KnownLHS.One & ~Imm);
20613 Known.Zero = IsVORR ? (KnownLHS.Zero & ~Imm) : (KnownLHS.Zero | Imm);
20614 break;
20615 }
20616 }
20617}
20618
20619static bool isLegalLogicalImmediate(unsigned Imm,
20620 const ARMSubtarget *Subtarget) {
20621 if (!Subtarget->isThumb())
20622 return ARM_AM::getSOImmVal(Imm) != -1;
20623 if (Subtarget->isThumb2())
20624 return ARM_AM::getT2SOImmVal(Imm) != -1;
20625 // Thumb1 only has 8-bit unsigned immediate.
20626 return Imm <= 255;
20627}
20628
20629/// Refine i32 AND/OR/XOR with a constant RHS using demanded bits: replace the
20630/// immediate with an equivalent constant that ARM/Thumb can encode as a
20631/// logical immediate (or that selects better lowering), without changing the
20632/// computed result on those demanded bits.
20633static bool optimizeLogicalImm(SDValue Op, unsigned Imm,
20634 const APInt &DemandedBits,
20635 const ARMSubtarget *Subtarget,
20637
20638 if (Imm == 0 || Imm == ~0U)
20639 return false;
20640
20641 unsigned Opc = Op.getOpcode();
20642 unsigned Demanded = DemandedBits.getZExtValue();
20643 EVT VT = Op.getValueType();
20644
20645 unsigned ShrunkImm = Imm & Demanded;
20646 unsigned ExpandedImm = Imm | ~Demanded;
20647
20648 auto IsLegalImm = [ShrunkImm, ExpandedImm](unsigned CandidateImm) -> bool {
20649 return (ShrunkImm & CandidateImm) == ShrunkImm &&
20650 (~ExpandedImm & CandidateImm) == 0;
20651 };
20652 auto UseImm = [Imm, Opc, Op, VT, &TLO](unsigned NewImm) -> bool {
20653 if (NewImm == Imm)
20654 return true;
20655 SDLoc DL(Op);
20656 SDValue NewC = TLO.DAG.getConstant(NewImm, DL, VT);
20657 SDValue NewOp =
20658 TLO.DAG.getNode(Opc, DL, VT, Op.getOperand(0), NewC, Op->getFlags());
20659 return TLO.CombineTo(Op, NewOp);
20660 };
20661
20662 // Shrunk immediate is 0: AND becomes zero; OR/XOR with 0 leaves the other
20663 // operand (still valid on demanded bits).
20664 if (ShrunkImm == 0) {
20665 ++NumOptimizedImms;
20666 return UseImm(ShrunkImm);
20667 }
20668
20669 // If the immediate is all ones: for AND this removes the operation; for
20670 // OR/XOR it remains a transform valid on demanded bits. (Target-independent
20671 // shrink may not fold this, so keep it to avoid obscure combine loops.)
20672 if (ExpandedImm == ~0U) {
20673 ++NumOptimizedImms;
20674 return UseImm(ExpandedImm);
20675 }
20676
20677 // Thumb1: prefer 0xFF / 0xFFFF when they fit the demanded-bit envelope so
20678 // lowering can match uxtb / uxth (AND immediates only; OR/XOR do not use
20679 // that). Run this before strict ShrunkImm: a tight 8-bit ShrunkImm can be
20680 // legal while 0xFF still matches the envelope and yields better isel (uxtb).
20681 if (Opc == ISD::AND && Subtarget->hasV6Ops()) {
20682 if (IsLegalImm(0xFF)) {
20683 ++NumOptimizedImms;
20684 return UseImm(0xFF);
20685 }
20686
20687 if (IsLegalImm(0xFFFF)) {
20688 ++NumOptimizedImms;
20689 return UseImm(0xFFFF);
20690 }
20691 }
20692
20693 // Don't optimize if it is legal.
20695 return false;
20696
20697 // FIXME: Check for BIC being legal causes infinite loop due to target
20698 // independent DAG combine undoing this.
20699
20700 // Prefer strict shrink when ShrunkImm encodes for this target, before
20701 // complement expansion.
20702 if (isLegalLogicalImmediate(ShrunkImm, Subtarget)) {
20703 ++NumOptimizedImms;
20704 return UseImm(ShrunkImm);
20705 }
20706
20707 // Complement expansion: if all undemanded bits are already one, ExpandedImm
20708 // is Imm with every non-demanded bit set. When (~ExpandedImm) < 256, the
20709 // complement fits in an 8-bit unsigned value, i.e. bits 8–31 of ExpandedImm
20710 // are all ones; only the low byte may differ from ~0. Use that expanded
20711 // constant so isel sees a mask shape that fits logical-immediate patterns.
20712 if ((~ExpandedImm) < 256) {
20713 ++NumOptimizedImms;
20714 return UseImm(ExpandedImm);
20715 }
20716
20717 // FIXME: The check for v6 is because this interferes with some ubfx
20718 // optimizations.
20719 if (Opc == ISD::AND && isLegalLogicalImmediate(~ExpandedImm, Subtarget) &&
20720 !Subtarget->hasV6Ops()) {
20721 ++NumOptimizedImms;
20722 return UseImm(ExpandedImm);
20723 }
20724
20725 // Potential improvements:
20726 //
20727 // We could try to recognize lsls+lsrs or lsrs+lsls pairs here.
20728 // We could try to prefer Thumb1 immediates which can be lowered to a
20729 // two-instruction sequence.
20730
20731 return false;
20732}
20733
20735 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
20736 TargetLoweringOpt &TLO) const {
20737 // Delay this optimization to as late as possible.
20738 if (!TLO.LegalOps)
20739 return false;
20740
20741 EVT VT = Op.getValueType();
20742
20743 // Ignore vectors.
20744 if (VT.isVector())
20745 return false;
20746
20747 unsigned Size = VT.getSizeInBits();
20748
20749 if (Size != 32)
20750 return false;
20751
20752 // Exit early if we demand all bits.
20753 if (DemandedBits.isAllOnes())
20754 return false;
20755
20756 switch (Op.getOpcode()) {
20757 default:
20758 return false;
20759 case ISD::AND:
20760 case ISD::OR:
20761 case ISD::XOR:
20762 break;
20763 }
20764 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
20765 if (!C)
20766 return false;
20767 unsigned Imm = C->getZExtValue();
20768 return optimizeLogicalImm(Op, Imm, DemandedBits, Subtarget, TLO);
20769}
20770
20772 SDValue Op, const APInt &OriginalDemandedBits,
20773 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
20774 unsigned Depth) const {
20775 unsigned Opc = Op.getOpcode();
20776
20777 switch (Opc) {
20778 case ARMISD::ASRL:
20779 case ARMISD::LSRL: {
20780 // If this is result 0 and the other result is unused, see if the demand
20781 // bits allow us to shrink this long shift into a standard small shift in
20782 // the opposite direction.
20783 if (Op.getResNo() == 0 && !Op->hasAnyUseOfValue(1) &&
20784 isa<ConstantSDNode>(Op->getOperand(2))) {
20785 unsigned ShAmt = Op->getConstantOperandVal(2);
20786 if (ShAmt < 32 && OriginalDemandedBits.isSubsetOf(APInt::getAllOnes(32)
20787 << (32 - ShAmt)))
20788 return TLO.CombineTo(
20789 Op, TLO.DAG.getNode(
20790 ISD::SHL, SDLoc(Op), MVT::i32, Op.getOperand(1),
20791 TLO.DAG.getConstant(32 - ShAmt, SDLoc(Op), MVT::i32)));
20792 }
20793 break;
20794 }
20795 case ARMISD::VBICIMM: {
20796 SDValue Op0 = Op.getOperand(0);
20797 unsigned ModImm = Op.getConstantOperandVal(1);
20798 unsigned EltBits = 0;
20799 uint64_t Mask = ARM_AM::decodeVMOVModImm(ModImm, EltBits);
20800 if ((OriginalDemandedBits & Mask) == 0)
20801 return TLO.CombineTo(Op, Op0);
20802 }
20803 }
20804
20806 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth);
20807}
20808
20809//===----------------------------------------------------------------------===//
20810// ARM Inline Assembly Support
20811//===----------------------------------------------------------------------===//
20812
20813const char *ARMTargetLowering::LowerXConstraint(EVT ConstraintVT) const {
20814 // At this point, we have to lower this constraint to something else, so we
20815 // lower it to an "r" or "w". However, by doing this we will force the result
20816 // to be in register, while the X constraint is much more permissive.
20817 //
20818 // Although we are correct (we are free to emit anything, without
20819 // constraints), we might break use cases that would expect us to be more
20820 // efficient and emit something else.
20821 if (!Subtarget->hasVFP2Base())
20822 return "r";
20823 if (ConstraintVT.isFloatingPoint())
20824 return "w";
20825 if (ConstraintVT.isVector() && Subtarget->hasNEON() &&
20826 (ConstraintVT.getSizeInBits() == 64 ||
20827 ConstraintVT.getSizeInBits() == 128))
20828 return "w";
20829
20830 return "r";
20831}
20832
20833/// getConstraintType - Given a constraint letter, return the type of
20834/// constraint it is for this target.
20837 unsigned S = Constraint.size();
20838 if (S == 1) {
20839 switch (Constraint[0]) {
20840 default: break;
20841 case 'l': return C_RegisterClass;
20842 case 'w': return C_RegisterClass;
20843 case 'h': return C_RegisterClass;
20844 case 'x': return C_RegisterClass;
20845 case 't': return C_RegisterClass;
20846 case 'j': return C_Immediate; // Constant for movw.
20847 // An address with a single base register. Due to the way we
20848 // currently handle addresses it is the same as an 'r' memory constraint.
20849 case 'Q': return C_Memory;
20850 }
20851 } else if (S == 2) {
20852 switch (Constraint[0]) {
20853 default: break;
20854 case 'T': return C_RegisterClass;
20855 // All 'U+' constraints are addresses.
20856 case 'U': return C_Memory;
20857 }
20858 }
20859 return TargetLowering::getConstraintType(Constraint);
20860}
20861
20862/// Examine constraint type and operand type and determine a weight value.
20863/// This object must already have been set up with the operand type
20864/// and the current alternative constraint selected.
20867 AsmOperandInfo &info, const char *constraint) const {
20869 Value *CallOperandVal = info.CallOperandVal;
20870 // If we don't have a value, we can't do a match,
20871 // but allow it at the lowest weight.
20872 if (!CallOperandVal)
20873 return CW_Default;
20874 Type *type = CallOperandVal->getType();
20875 // Look at the constraint type.
20876 switch (*constraint) {
20877 default:
20879 break;
20880 case 'l':
20881 if (type->isIntegerTy()) {
20882 if (Subtarget->isThumb())
20884 else
20886 }
20887 break;
20888 case 'w':
20889 if (type->isFloatingPointTy())
20891 break;
20892 }
20893 return weight;
20894}
20895
20896static bool isIncompatibleReg(const MCPhysReg &PR, MVT VT) {
20897 if (PR == 0 || VT == MVT::Other)
20898 return false;
20899 if (ARM::SPRRegClass.contains(PR))
20900 return VT != MVT::f32 && VT != MVT::f16 && VT != MVT::i32;
20901 if (ARM::DPRRegClass.contains(PR))
20902 return VT != MVT::f64 && !VT.is64BitVector();
20903 return false;
20904}
20905
20906using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20907
20909 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
20910 switch (Constraint.size()) {
20911 case 1:
20912 // GCC ARM Constraint Letters
20913 switch (Constraint[0]) {
20914 case 'l': // Low regs or general regs.
20915 if (Subtarget->isThumb())
20916 return RCPair(0U, &ARM::tGPRRegClass);
20917 return RCPair(0U, &ARM::GPRRegClass);
20918 case 'h': // High regs or no regs.
20919 if (Subtarget->isThumb())
20920 return RCPair(0U, &ARM::hGPRRegClass);
20921 break;
20922 case 'r':
20923 if (Subtarget->isThumb1Only())
20924 return RCPair(0U, &ARM::tGPRRegClass);
20925 return RCPair(0U, &ARM::GPRRegClass);
20926 case 'w':
20927 if (VT == MVT::Other)
20928 break;
20929 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20930 return RCPair(0U, &ARM::SPRRegClass);
20931 if (VT.getSizeInBits() == 64)
20932 return RCPair(0U, &ARM::DPRRegClass);
20933 if (VT.getSizeInBits() == 128)
20934 return RCPair(0U, &ARM::QPRRegClass);
20935 break;
20936 case 'x':
20937 if (VT == MVT::Other)
20938 break;
20939 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20940 return RCPair(0U, &ARM::SPR_8RegClass);
20941 if (VT.getSizeInBits() == 64)
20942 return RCPair(0U, &ARM::DPR_8RegClass);
20943 if (VT.getSizeInBits() == 128)
20944 return RCPair(0U, &ARM::QPR_8RegClass);
20945 break;
20946 case 't':
20947 if (VT == MVT::Other)
20948 break;
20949 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20950 return RCPair(0U, &ARM::SPRRegClass);
20951 if (VT.getSizeInBits() == 64)
20952 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20953 if (VT.getSizeInBits() == 128)
20954 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20955 break;
20956 }
20957 break;
20958
20959 case 2:
20960 if (Constraint[0] == 'T') {
20961 switch (Constraint[1]) {
20962 default:
20963 break;
20964 case 'e':
20965 return RCPair(0U, &ARM::tGPREvenRegClass);
20966 case 'o':
20967 return RCPair(0U, &ARM::tGPROddRegClass);
20968 }
20969 }
20970 break;
20971
20972 default:
20973 break;
20974 }
20975
20976 if (StringRef("{cc}").equals_insensitive(Constraint))
20977 return std::make_pair(unsigned(ARM::CPSR), &ARM::CCRRegClass);
20978
20979 // r14 is an alias of lr.
20980 if (StringRef("{r14}").equals_insensitive(Constraint))
20981 Constraint = "{lr}";
20982
20983 auto RCP = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20984 if (isIncompatibleReg(RCP.first, VT))
20985 return {0, nullptr};
20986 return RCP;
20987}
20988
20989/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
20990/// vector. If it is invalid, don't add anything to Ops.
20992 StringRef Constraint,
20993 std::vector<SDValue> &Ops,
20994 SelectionDAG &DAG) const {
20995 SDValue Result;
20996
20997 // Currently only support length 1 constraints.
20998 if (Constraint.size() != 1)
20999 return;
21000
21001 char ConstraintLetter = Constraint[0];
21002 switch (ConstraintLetter) {
21003 default: break;
21004 case 'j':
21005 case 'I': case 'J': case 'K': case 'L':
21006 case 'M': case 'N': case 'O':
21008 if (!C)
21009 return;
21010
21011 int64_t CVal64 = C->getSExtValue();
21012 int CVal = (int) CVal64;
21013 // None of these constraints allow values larger than 32 bits. Check
21014 // that the value fits in an int.
21015 if (CVal != CVal64)
21016 return;
21017
21018 switch (ConstraintLetter) {
21019 case 'j':
21020 // Constant suitable for movw, must be between 0 and
21021 // 65535.
21022 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
21023 if (CVal >= 0 && CVal <= 65535)
21024 break;
21025 return;
21026 case 'I':
21027 if (Subtarget->isThumb1Only()) {
21028 // This must be a constant between 0 and 255, for ADD
21029 // immediates.
21030 if (CVal >= 0 && CVal <= 255)
21031 break;
21032 } else if (Subtarget->isThumb2()) {
21033 // A constant that can be used as an immediate value in a
21034 // data-processing instruction.
21035 if (ARM_AM::getT2SOImmVal(CVal) != -1)
21036 break;
21037 } else {
21038 // A constant that can be used as an immediate value in a
21039 // data-processing instruction.
21040 if (ARM_AM::getSOImmVal(CVal) != -1)
21041 break;
21042 }
21043 return;
21044
21045 case 'J':
21046 if (Subtarget->isThumb1Only()) {
21047 // This must be a constant between -255 and -1, for negated ADD
21048 // immediates. This can be used in GCC with an "n" modifier that
21049 // prints the negated value, for use with SUB instructions. It is
21050 // not useful otherwise but is implemented for compatibility.
21051 if (CVal >= -255 && CVal <= -1)
21052 break;
21053 } else {
21054 // This must be a constant between -4095 and 4095. This is suitable
21055 // for use as the immediate offset field in LDR and STR instructions
21056 // such as LDR r0,[r1,#offset].
21057 if (CVal >= -4095 && CVal <= 4095)
21058 break;
21059 }
21060 return;
21061
21062 case 'K':
21063 if (Subtarget->isThumb1Only()) {
21064 // A 32-bit value where only one byte has a nonzero value. Exclude
21065 // zero to match GCC. This constraint is used by GCC internally for
21066 // constants that can be loaded with a move/shift combination.
21067 // It is not useful otherwise but is implemented for compatibility.
21068 if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(CVal))
21069 break;
21070 } else if (Subtarget->isThumb2()) {
21071 // A constant whose bitwise inverse can be used as an immediate
21072 // value in a data-processing instruction. This can be used in GCC
21073 // with a "B" modifier that prints the inverted value, for use with
21074 // BIC and MVN instructions. It is not useful otherwise but is
21075 // implemented for compatibility.
21076 if (ARM_AM::getT2SOImmVal(~CVal) != -1)
21077 break;
21078 } else {
21079 // A constant whose bitwise inverse can be used as an immediate
21080 // value in a data-processing instruction. This can be used in GCC
21081 // with a "B" modifier that prints the inverted value, for use with
21082 // BIC and MVN instructions. It is not useful otherwise but is
21083 // implemented for compatibility.
21084 if (ARM_AM::getSOImmVal(~CVal) != -1)
21085 break;
21086 }
21087 return;
21088
21089 case 'L':
21090 if (Subtarget->isThumb1Only()) {
21091 // This must be a constant between -7 and 7,
21092 // for 3-operand ADD/SUB immediate instructions.
21093 if (CVal >= -7 && CVal < 7)
21094 break;
21095 } else if (Subtarget->isThumb2()) {
21096 // A constant whose negation can be used as an immediate value in a
21097 // data-processing instruction. This can be used in GCC with an "n"
21098 // modifier that prints the negated value, for use with SUB
21099 // instructions. It is not useful otherwise but is implemented for
21100 // compatibility.
21101 if (ARM_AM::getT2SOImmVal(-CVal) != -1)
21102 break;
21103 } else {
21104 // A constant whose negation can be used as an immediate value in a
21105 // data-processing instruction. This can be used in GCC with an "n"
21106 // modifier that prints the negated value, for use with SUB
21107 // instructions. It is not useful otherwise but is implemented for
21108 // compatibility.
21109 if (ARM_AM::getSOImmVal(-CVal) != -1)
21110 break;
21111 }
21112 return;
21113
21114 case 'M':
21115 if (Subtarget->isThumb1Only()) {
21116 // This must be a multiple of 4 between 0 and 1020, for
21117 // ADD sp + immediate.
21118 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
21119 break;
21120 } else {
21121 // A power of two or a constant between 0 and 32. This is used in
21122 // GCC for the shift amount on shifted register operands, but it is
21123 // useful in general for any shift amounts.
21124 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
21125 break;
21126 }
21127 return;
21128
21129 case 'N':
21130 if (Subtarget->isThumb1Only()) {
21131 // This must be a constant between 0 and 31, for shift amounts.
21132 if (CVal >= 0 && CVal <= 31)
21133 break;
21134 }
21135 return;
21136
21137 case 'O':
21138 if (Subtarget->isThumb1Only()) {
21139 // This must be a multiple of 4 between -508 and 508, for
21140 // ADD/SUB sp = sp + immediate.
21141 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
21142 break;
21143 }
21144 return;
21145 }
21146 Result = DAG.getSignedTargetConstant(CVal, SDLoc(Op), Op.getValueType());
21147 break;
21148 }
21149
21150 if (Result.getNode()) {
21151 Ops.push_back(Result);
21152 return;
21153 }
21154 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
21155}
21156
21157static RTLIB::Libcall getDivRemLibcall(
21158 const SDNode *N, MVT::SimpleValueType SVT) {
21159 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
21160 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
21161 "Unhandled Opcode in getDivRemLibcall");
21162 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
21163 N->getOpcode() == ISD::SREM;
21164 RTLIB::Libcall LC;
21165 switch (SVT) {
21166 default: llvm_unreachable("Unexpected request for libcall!");
21167 case MVT::i8: LC = isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
21168 case MVT::i16: LC = isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
21169 case MVT::i32: LC = isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
21170 case MVT::i64: LC = isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
21171 }
21172 return LC;
21173}
21174
21177 const AttributeList &FuncAttrs, RTLIB::LibcallImpl LCImpl) {
21178 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
21179 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
21180 "Unhandled Opcode in getDivRemArgList");
21181 SDValue Ops[2] = {N->getOperand(0), N->getOperand(1)};
21182
21183 // The Windows __rt_*div* helpers take the divisor before the dividend.
21184 switch (LCImpl) {
21185 case RTLIB::impl___rt_sdiv:
21186 case RTLIB::impl___rt_udiv:
21187 case RTLIB::impl___rt_sdiv64:
21188 case RTLIB::impl___rt_udiv64:
21189 std::swap(Ops[0], Ops[1]);
21190 break;
21191 default:
21192 break;
21193 }
21194
21195 return TargetLowering::getArgListForFunctionType(FuncTy, FuncAttrs, Ops);
21196}
21197
21198SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
21199 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
21200 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
21201 Subtarget->isTargetFuchsia() || Subtarget->isTargetWindows()) &&
21202 "Register-based DivRem lowering only");
21203 unsigned Opcode = Op->getOpcode();
21204 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) &&
21205 "Invalid opcode for Div/Rem lowering");
21206 bool isSigned = (Opcode == ISD::SDIVREM);
21207 EVT VT = Op->getValueType(0);
21208 SDLoc dl(Op);
21209
21210 if (VT == MVT::i64 && isa<ConstantSDNode>(Op.getOperand(1))) {
21212 if (expandDIVREMByConstant(Op.getNode(), Result, MVT::i32, DAG)) {
21213 SDValue Res0 =
21214 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[0], Result[1]);
21215 SDValue Res1 =
21216 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[2], Result[3]);
21217 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
21218 {Res0, Res1});
21219 }
21220 }
21221
21222 // If the target has hardware divide, use divide + multiply + subtract:
21223 // div = a / b
21224 // rem = a - b * div
21225 // return {div, rem}
21226 // This should be lowered into UDIV/SDIV + MLS later on.
21227 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
21228 : Subtarget->hasDivideInARMMode();
21229 if (hasDivide && Op->getValueType(0).isSimple() &&
21230 Op->getSimpleValueType(0) == MVT::i32) {
21231 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
21232 const SDValue Dividend = Op->getOperand(0);
21233 const SDValue Divisor = Op->getOperand(1);
21234 SDValue Div = DAG.getNode(DivOpcode, dl, VT, Dividend, Divisor);
21235 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor);
21236 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
21237
21238 SDValue Values[2] = {Div, Rem};
21239 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VT, VT), Values);
21240 }
21241
21242 RTLIB::Libcall LC = getDivRemLibcall(Op.getNode(),
21243 VT.getSimpleVT().SimpleTy);
21244 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
21245 if (LCImpl == RTLIB::Unsupported)
21246 return SDValue();
21247
21248 auto [FuncTy, FuncAttrs] =
21250 *DAG.getContext(), getTM().getTargetTriple(), DAG.getDataLayout(),
21251 LCImpl);
21252 Type *RetTy = FuncTy->getReturnType();
21253
21254 SDValue InChain = DAG.getEntryNode();
21255
21257 getDivRemArgList(Op.getNode(), FuncTy, FuncAttrs, LCImpl);
21258
21259 SDValue Callee =
21260 DAG.getExternalSymbol(LCImpl, getPointerTy(DAG.getDataLayout()));
21261
21262 if (getTM().getTargetTriple().isOSWindows())
21263 InChain = WinDBZCheckDenominator(DAG, Op.getNode(), InChain);
21264
21265 TargetLowering::CallLoweringInfo CLI(DAG);
21266 CLI.setDebugLoc(dl)
21267 .setChain(InChain)
21268 .setCallee(DAG.getLibcalls().getLibcallImplCallingConv(LCImpl), RetTy,
21269 Callee, std::move(Args))
21270 .setInRegister()
21273
21274 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
21275 return CallInfo.first;
21276}
21277
21278// Lowers REM using divmod helpers
21279// see RTABI section 4.2/4.3
21280SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
21281 EVT VT = N->getValueType(0);
21282
21283 if (VT == MVT::i64 && isa<ConstantSDNode>(N->getOperand(1))) {
21285 if (expandDIVREMByConstant(N, Result, MVT::i32, DAG))
21286 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), N->getValueType(0),
21287 Result[0], Result[1]);
21288 }
21289
21290 RTLIB::Libcall LC = getDivRemLibcall(N, N->getValueType(0).getSimpleVT().
21291 SimpleTy);
21292 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
21293 if (LCImpl == RTLIB::Unsupported)
21294 return SDValue();
21295
21296 auto [FuncTy, FuncAttrs] =
21298 *DAG.getContext(), getTM().getTargetTriple(), DAG.getDataLayout(),
21299 LCImpl);
21300 Type *RetTy = FuncTy->getReturnType();
21301
21302 SDValue InChain = DAG.getEntryNode();
21304 getDivRemArgList(N, FuncTy, FuncAttrs, LCImpl);
21305 bool isSigned = N->getOpcode() == ISD::SREM;
21306
21307 SDValue Callee =
21308 DAG.getExternalSymbol(LCImpl, getPointerTy(DAG.getDataLayout()));
21309
21310 if (getTM().getTargetTriple().isOSWindows())
21311 InChain = WinDBZCheckDenominator(DAG, N, InChain);
21312
21313 // Lower call
21314 CallLoweringInfo CLI(DAG);
21315 CLI.setChain(InChain)
21316 .setCallee(DAG.getLibcalls().getLibcallImplCallingConv(LCImpl), RetTy,
21317 Callee, std::move(Args))
21320 .setDebugLoc(SDLoc(N));
21321 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
21322
21323 // Return second (rem) result operand (first contains div)
21324 SDNode *ResNode = CallResult.first.getNode();
21325 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands");
21326 return ResNode->getOperand(1);
21327}
21328
21329SDValue
21330ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
21331 assert(getTM().getTargetTriple().isOSWindows() &&
21332 "unsupported target platform");
21333 SDLoc DL(Op);
21334
21335 // Get the inputs.
21336 SDValue Chain = Op.getOperand(0);
21337 SDValue Size = Op.getOperand(1);
21338
21340 "no-stack-arg-probe")) {
21341 MaybeAlign Align =
21342 cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
21343 SDValue SP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
21344 Chain = SP.getValue(1);
21345 SP = DAG.getNode(ISD::SUB, DL, MVT::i32, SP, Size);
21346 if (Align)
21347 SP = DAG.getNode(ISD::AND, DL, MVT::i32, SP.getValue(0),
21348 DAG.getSignedConstant(-Align->value(), DL, MVT::i32));
21349 Chain = DAG.getCopyToReg(Chain, DL, ARM::SP, SP);
21350 SDValue Ops[2] = { SP, Chain };
21351 return DAG.getMergeValues(Ops, DL);
21352 }
21353
21354 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size,
21355 DAG.getConstant(2, DL, MVT::i32));
21356
21357 SDValue Glue;
21358 Chain = DAG.getCopyToReg(Chain, DL, ARM::R4, Words, Glue);
21359 Glue = Chain.getValue(1);
21360
21361 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
21362 Chain = DAG.getNode(ARMISD::WIN__CHKSTK, DL, NodeTys, Chain, Glue);
21363
21364 SDValue NewSP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
21365 Chain = NewSP.getValue(1);
21366
21367 SDValue Ops[2] = { NewSP, Chain };
21368 return DAG.getMergeValues(Ops, DL);
21369}
21370
21371SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
21372 bool IsStrict = Op->isStrictFPOpcode();
21373 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
21374 EVT DstTy = Op.getValueType();
21375 const unsigned DstSz = DstTy.getSizeInBits();
21376 EVT SrcTy = SrcVal.getValueType();
21377 const unsigned SrcSz = SrcTy.getSizeInBits();
21378 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
21379 "Unexpected type for custom-lowering FP_EXTEND");
21380
21381 // f16->f32 is legal with hasFP16
21382 if (DstTy == MVT::f32 && SrcTy == MVT::f16 && Subtarget->hasFP16())
21383 return Op;
21384 // f16->f64 is legal with hasFP16 and hasFP64
21385 if (DstTy == MVT::f64 && SrcTy == MVT::f16 && Subtarget->hasFP16() &&
21386 Subtarget->hasFP64() && Subtarget->hasFPARMv8Base())
21387 return Op;
21388
21389 // Expand any bf16 ops into a shift and a possible further extend
21390 if (SrcTy == MVT::bf16) {
21391 SDLoc DL(Op);
21392 SDValue Mov = DAG.getNode(ARMISD::VMOVrs, DL, MVT::i32, SrcVal);
21393 SDValue Shft = DAG.getNode(ISD::SHL, DL, MVT::i32, Mov,
21394 DAG.getConstant(16, DL, MVT::i32));
21395 SDValue Res = DAG.getBitcast(MVT::f32, Shft);
21396 if (DstTy == MVT::f32)
21397 return IsStrict ? DAG.getMergeValues({Res, Op.getOperand(0)}, DL) : Res;
21398
21399 assert(DstTy == MVT::f64 && "Unexpected BF16 convert");
21400 if (IsStrict)
21401 return DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {DstTy, MVT::Other},
21402 {Op.getOperand(0), Res});
21403 return DAG.getNode(ISD::FP_EXTEND, DL, DstTy, Res);
21404 }
21405
21406 // Converting from 32 -> 64 is valid if we have FP64.
21407 if (SrcTy == MVT::f32 && DstTy == MVT::f64 && Subtarget->hasFP64()) {
21408 // FIXME: Remove this when we have strict fp instruction selection patterns
21409 if (IsStrict) {
21410 SDLoc Loc(Op);
21411 SDValue Result = DAG.getNode(ISD::FP_EXTEND, Loc, DstTy, SrcVal);
21412 return DAG.getMergeValues({Result, Op.getOperand(0)}, Loc);
21413 }
21414 return Op;
21415 }
21416
21417 // Either we are converting from 16 -> 64, without FP16 and/or
21418 // FP.double-precision or without Armv8-fp. So we must do it in two
21419 // steps.
21420 // Or we are converting from 32 -> 64 without fp.double-precision or 16 -> 32
21421 // without FP16. So we must do a function call.
21422 SDLoc Loc(Op);
21423 RTLIB::Libcall LC;
21424 MakeLibCallOptions CallOptions;
21425 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21426 for (unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
21427 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
21428 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
21429 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
21430 if (Supported) {
21431 if (IsStrict) {
21432 SrcVal = DAG.getNode(ISD::STRICT_FP_EXTEND, Loc,
21433 {DstVT, MVT::Other}, {Chain, SrcVal});
21434 Chain = SrcVal.getValue(1);
21435 } else {
21436 SrcVal = DAG.getNode(ISD::FP_EXTEND, Loc, DstVT, SrcVal);
21437 }
21438 } else {
21439 LC = RTLIB::getFPEXT(SrcVT, DstVT);
21440 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
21441 "Unexpected type for custom-lowering FP_EXTEND");
21442 std::tie(SrcVal, Chain) = makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
21443 Loc, Chain);
21444 }
21445 }
21446
21447 return IsStrict ? DAG.getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
21448}
21449
21450SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
21451 bool IsStrict = Op->isStrictFPOpcode();
21452
21453 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
21454 EVT SrcVT = SrcVal.getValueType();
21455 EVT DstVT = Op.getValueType();
21456
21457 if (DstVT == MVT::bf16) {
21458 if (Subtarget->hasBF16() && SrcVT == MVT::f32)
21459 return Op;
21460 return SDValue();
21461 }
21462
21463 const unsigned DstSz = Op.getValueType().getSizeInBits();
21464 const unsigned SrcSz = SrcVT.getSizeInBits();
21465 (void)DstSz;
21466 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
21467 "Unexpected type for custom-lowering FP_ROUND");
21468
21469 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&
21470 "With both FP DP and 16, any FP conversion is legal!");
21471
21472 SDLoc Loc(Op);
21473
21474 // Instruction from 32 -> 16 if hasFP16 is valid
21475 if (SrcSz == 32 && Subtarget->hasFP16())
21476 return Op;
21477
21478 // Lib call from 32 -> 16 / 64 -> [32, 16]
21479 RTLIB::Libcall LC = RTLIB::getFPROUND(SrcVT, DstVT);
21480 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
21481 "Unexpected type for custom-lowering FP_ROUND");
21482 MakeLibCallOptions CallOptions;
21483 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21485 std::tie(Result, Chain) = makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
21486 Loc, Chain);
21487 return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
21488}
21489
21490bool
21492 // The ARM target isn't yet aware of offsets.
21493 return false;
21494}
21495
21497 if (v == 0xffffffff)
21498 return false;
21499
21500 // there can be 1's on either or both "outsides", all the "inside"
21501 // bits must be 0's
21502 return isShiftedMask_32(~v);
21503}
21504
21505/// Returns how to materialize the specified FP immediate.
21508 bool ForCodeSize) const {
21509 if (!Subtarget->hasVFP3Base())
21511 if (VT == MVT::f16 && Subtarget->hasFullFP16())
21514 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
21516 return FPImmAction::Legal;
21517 if (VT == MVT::f32)
21520 if (VT == MVT::f64 && Subtarget->hasFP64())
21524}
21525
21528 bool ForCodeSize) const {
21529 if (Subtarget->useSoftFloat() || !Subtarget->hasVFP2Base())
21530 return FPImmAction::Bitcast;
21532}
21533
21534/// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
21535/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
21536/// specified in the intrinsic calls.
21539 MachineFunction &MF, unsigned Intrinsic) const {
21540 IntrinsicInfo Info;
21541 switch (Intrinsic) {
21542 case Intrinsic::arm_neon_vld1:
21543 case Intrinsic::arm_neon_vld2:
21544 case Intrinsic::arm_neon_vld3:
21545 case Intrinsic::arm_neon_vld4:
21546 case Intrinsic::arm_neon_vld2lane:
21547 case Intrinsic::arm_neon_vld3lane:
21548 case Intrinsic::arm_neon_vld4lane:
21549 case Intrinsic::arm_neon_vld2dup:
21550 case Intrinsic::arm_neon_vld3dup:
21551 case Intrinsic::arm_neon_vld4dup: {
21552 Info.opc = ISD::INTRINSIC_W_CHAIN;
21553 // Conservatively set memVT to the entire set of vectors loaded.
21554 auto &DL = I.getDataLayout();
21555 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
21556 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
21557 Info.ptrVal = I.getArgOperand(0);
21558 Info.offset = 0;
21559 Value *AlignArg = I.getArgOperand(I.arg_size() - 1);
21560 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21561 // volatile loads with NEON intrinsics not supported
21562 Info.flags = MachineMemOperand::MOLoad;
21563 Infos.push_back(Info);
21564 return;
21565 }
21566 case Intrinsic::arm_neon_vld1x2:
21567 case Intrinsic::arm_neon_vld1x3:
21568 case Intrinsic::arm_neon_vld1x4: {
21569 Info.opc = ISD::INTRINSIC_W_CHAIN;
21570 // Conservatively set memVT to the entire set of vectors loaded.
21571 auto &DL = I.getDataLayout();
21572 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
21573 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
21574 Info.ptrVal = I.getArgOperand(I.arg_size() - 1);
21575 Info.offset = 0;
21576 Info.align = I.getParamAlign(I.arg_size() - 1).valueOrOne();
21577 // volatile loads with NEON intrinsics not supported
21578 Info.flags = MachineMemOperand::MOLoad;
21579 Infos.push_back(Info);
21580 return;
21581 }
21582 case Intrinsic::arm_neon_vst1:
21583 case Intrinsic::arm_neon_vst2:
21584 case Intrinsic::arm_neon_vst3:
21585 case Intrinsic::arm_neon_vst4:
21586 case Intrinsic::arm_neon_vst2lane:
21587 case Intrinsic::arm_neon_vst3lane:
21588 case Intrinsic::arm_neon_vst4lane: {
21589 Info.opc = ISD::INTRINSIC_VOID;
21590 // Conservatively set memVT to the entire set of vectors stored.
21591 auto &DL = I.getDataLayout();
21592 unsigned NumElts = 0;
21593 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
21594 Type *ArgTy = I.getArgOperand(ArgI)->getType();
21595 if (!ArgTy->isVectorTy())
21596 break;
21597 NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
21598 }
21599 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
21600 Info.ptrVal = I.getArgOperand(0);
21601 Info.offset = 0;
21602 Value *AlignArg = I.getArgOperand(I.arg_size() - 1);
21603 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21604 // volatile stores with NEON intrinsics not supported
21605 Info.flags = MachineMemOperand::MOStore;
21606 Infos.push_back(Info);
21607 return;
21608 }
21609 case Intrinsic::arm_neon_vst1x2:
21610 case Intrinsic::arm_neon_vst1x3:
21611 case Intrinsic::arm_neon_vst1x4: {
21612 Info.opc = ISD::INTRINSIC_VOID;
21613 // Conservatively set memVT to the entire set of vectors stored.
21614 auto &DL = I.getDataLayout();
21615 unsigned NumElts = 0;
21616 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
21617 Type *ArgTy = I.getArgOperand(ArgI)->getType();
21618 if (!ArgTy->isVectorTy())
21619 break;
21620 NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
21621 }
21622 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
21623 Info.ptrVal = I.getArgOperand(0);
21624 Info.offset = 0;
21625 Info.align = I.getParamAlign(0).valueOrOne();
21626 // volatile stores with NEON intrinsics not supported
21627 Info.flags = MachineMemOperand::MOStore;
21628 Infos.push_back(Info);
21629 return;
21630 }
21631 case Intrinsic::arm_mve_vld2q:
21632 case Intrinsic::arm_mve_vld4q: {
21633 Info.opc = ISD::INTRINSIC_W_CHAIN;
21634 // Conservatively set memVT to the entire set of vectors loaded.
21635 Type *VecTy = cast<StructType>(I.getType())->getElementType(1);
21636 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21637 Info.memVT = EVT::getVectorVT(VecTy->getContext(), MVT::i64, Factor * 2);
21638 Info.ptrVal = I.getArgOperand(0);
21639 Info.offset = 0;
21640 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
21641 // volatile loads with MVE intrinsics not supported
21642 Info.flags = MachineMemOperand::MOLoad;
21643 Infos.push_back(Info);
21644 return;
21645 }
21646 case Intrinsic::arm_mve_vst2q:
21647 case Intrinsic::arm_mve_vst4q: {
21648 Info.opc = ISD::INTRINSIC_VOID;
21649 // Conservatively set memVT to the entire set of vectors stored.
21650 Type *VecTy = I.getArgOperand(1)->getType();
21651 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21652 Info.memVT = EVT::getVectorVT(VecTy->getContext(), MVT::i64, Factor * 2);
21653 Info.ptrVal = I.getArgOperand(0);
21654 Info.offset = 0;
21655 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
21656 // volatile stores with MVE intrinsics not supported
21657 Info.flags = MachineMemOperand::MOStore;
21658 Infos.push_back(Info);
21659 return;
21660 }
21661 case Intrinsic::arm_mve_vldr_gather_base:
21662 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21663 Info.opc = ISD::INTRINSIC_W_CHAIN;
21664 Info.ptrVal = nullptr;
21665 Info.memVT = MVT::getVT(I.getType());
21666 Info.align = Align(1);
21667 Info.flags |= MachineMemOperand::MOLoad;
21668 Infos.push_back(Info);
21669 return;
21670 }
21671 case Intrinsic::arm_mve_vldr_gather_base_wb:
21672 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21673 Info.opc = ISD::INTRINSIC_W_CHAIN;
21674 Info.ptrVal = nullptr;
21675 Info.memVT = MVT::getVT(I.getType()->getContainedType(0));
21676 Info.align = Align(1);
21677 Info.flags |= MachineMemOperand::MOLoad;
21678 Infos.push_back(Info);
21679 return;
21680 }
21681 case Intrinsic::arm_mve_vldr_gather_offset:
21682 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21683 Info.opc = ISD::INTRINSIC_W_CHAIN;
21684 Info.ptrVal = nullptr;
21685 MVT DataVT = MVT::getVT(I.getType());
21686 unsigned MemSize = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
21687 Info.memVT = MVT::getVectorVT(MVT::getIntegerVT(MemSize),
21688 DataVT.getVectorNumElements());
21689 Info.align = Align(1);
21690 Info.flags |= MachineMemOperand::MOLoad;
21691 Infos.push_back(Info);
21692 return;
21693 }
21694 case Intrinsic::arm_mve_vstr_scatter_base:
21695 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21696 Info.opc = ISD::INTRINSIC_VOID;
21697 Info.ptrVal = nullptr;
21698 Info.memVT = MVT::getVT(I.getArgOperand(2)->getType());
21699 Info.align = Align(1);
21700 Info.flags |= MachineMemOperand::MOStore;
21701 Infos.push_back(Info);
21702 return;
21703 }
21704 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21705 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21706 Info.opc = ISD::INTRINSIC_W_CHAIN;
21707 Info.ptrVal = nullptr;
21708 Info.memVT = MVT::getVT(I.getArgOperand(2)->getType());
21709 Info.align = Align(1);
21710 Info.flags |= MachineMemOperand::MOStore;
21711 Infos.push_back(Info);
21712 return;
21713 }
21714 case Intrinsic::arm_mve_vstr_scatter_offset:
21715 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21716 Info.opc = ISD::INTRINSIC_VOID;
21717 Info.ptrVal = nullptr;
21718 MVT DataVT = MVT::getVT(I.getArgOperand(2)->getType());
21719 unsigned MemSize = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
21720 Info.memVT = MVT::getVectorVT(MVT::getIntegerVT(MemSize),
21721 DataVT.getVectorNumElements());
21722 Info.align = Align(1);
21723 Info.flags |= MachineMemOperand::MOStore;
21724 Infos.push_back(Info);
21725 return;
21726 }
21727 case Intrinsic::arm_ldaex:
21728 case Intrinsic::arm_ldrex: {
21729 auto &DL = I.getDataLayout();
21730 Type *ValTy = I.getParamElementType(0);
21731 Info.opc = ISD::INTRINSIC_W_CHAIN;
21732 Info.memVT = MVT::getVT(ValTy);
21733 Info.ptrVal = I.getArgOperand(0);
21734 Info.offset = 0;
21735 Info.align = DL.getABITypeAlign(ValTy);
21737 Infos.push_back(Info);
21738 return;
21739 }
21740 case Intrinsic::arm_stlex:
21741 case Intrinsic::arm_strex: {
21742 auto &DL = I.getDataLayout();
21743 Type *ValTy = I.getParamElementType(1);
21744 Info.opc = ISD::INTRINSIC_W_CHAIN;
21745 Info.memVT = MVT::getVT(ValTy);
21746 Info.ptrVal = I.getArgOperand(1);
21747 Info.offset = 0;
21748 Info.align = DL.getABITypeAlign(ValTy);
21750 Infos.push_back(Info);
21751 return;
21752 }
21753 case Intrinsic::arm_stlexd:
21754 case Intrinsic::arm_strexd:
21755 Info.opc = ISD::INTRINSIC_W_CHAIN;
21756 Info.memVT = MVT::i64;
21757 Info.ptrVal = I.getArgOperand(2);
21758 Info.offset = 0;
21759 Info.align = Align(8);
21761 Infos.push_back(Info);
21762 return;
21763
21764 case Intrinsic::arm_ldaexd:
21765 case Intrinsic::arm_ldrexd:
21766 Info.opc = ISD::INTRINSIC_W_CHAIN;
21767 Info.memVT = MVT::i64;
21768 Info.ptrVal = I.getArgOperand(0);
21769 Info.offset = 0;
21770 Info.align = Align(8);
21772 Infos.push_back(Info);
21773 return;
21774
21775 default:
21776 break;
21777 }
21778}
21779
21780/// Returns true if it is beneficial to convert a load of a constant
21781/// to just the constant itself.
21783 Type *Ty) const {
21784 assert(Ty->isIntegerTy());
21785
21786 unsigned Bits = Ty->getPrimitiveSizeInBits();
21787 if (Bits == 0 || Bits > 32)
21788 return false;
21789 return true;
21790}
21791
21794 unsigned Index) const {
21797
21798 if (Index == 0 || Index == ResVT.getVectorNumElements())
21801}
21802
21804 ARM_MB::MemBOpt Domain) const {
21805 // First, if the target has no DMB, see what fallback we can use.
21806 if (!Subtarget->hasDataBarrier()) {
21807 // Some ARMv6 cpus can support data barriers with an mcr instruction.
21808 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
21809 // here.
21810 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21811 Value* args[6] = {Builder.getInt32(15), Builder.getInt32(0),
21812 Builder.getInt32(0), Builder.getInt32(7),
21813 Builder.getInt32(10), Builder.getInt32(5)};
21814 return Builder.CreateIntrinsicWithoutFolding(Intrinsic::arm_mcr, args);
21815 }
21816 // Instead of using barriers, atomic accesses on these subtargets use
21817 // libcalls.
21818 llvm_unreachable("makeDMB on a target so old that it has no barriers");
21819 } else {
21820 // Only a full system barrier exists in the M-class architectures.
21821 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
21822 Constant *CDomain = Builder.getInt32(Domain);
21823 return Builder.CreateIntrinsicWithoutFolding(Intrinsic::arm_dmb, CDomain);
21824 }
21825}
21826
21827// Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
21829 Instruction *Inst,
21830 AtomicOrdering Ord) const {
21831 switch (Ord) {
21834 llvm_unreachable("Invalid fence: unordered/non-atomic");
21837 return nullptr; // Nothing to do
21839 if (!Inst->hasAtomicStore())
21840 return nullptr; // Nothing to do
21841 [[fallthrough]];
21844 if (Subtarget->preferISHSTBarriers())
21845 return makeDMB(Builder, ARM_MB::ISHST);
21846 // FIXME: add a comment with a link to documentation justifying this.
21847 else
21848 return makeDMB(Builder, ARM_MB::ISH);
21849 }
21850 llvm_unreachable("Unknown fence ordering in emitLeadingFence");
21851}
21852
21854 Instruction *Inst,
21855 AtomicOrdering Ord) const {
21856 switch (Ord) {
21859 llvm_unreachable("Invalid fence: unordered/not-atomic");
21862 return nullptr; // Nothing to do
21866 return makeDMB(Builder, ARM_MB::ISH);
21867 }
21868 llvm_unreachable("Unknown fence ordering in emitTrailingFence");
21869}
21870
21871// Loads and stores less than 64-bits are already atomic; ones above that
21872// are doomed anyway, so defer to the default libcall and blame the OS when
21873// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21874// anything for those.
21877 bool has64BitLLSC;
21878 if (Subtarget->isMClass())
21879 has64BitLLSC = false;
21880 else if (Subtarget->isThumb())
21881 has64BitLLSC = Subtarget->hasV7Ops();
21882 else
21883 has64BitLLSC = Subtarget->hasV6Ops();
21884
21885 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21886 if (Size == 64 && has64BitLLSC)
21888
21889 if (Size <= 32 && !hasNativeAtomics(*Subtarget) && !InsertFencesForAtomic &&
21890 isReleaseOrStronger(SI->getOrdering()))
21892
21894}
21895
21896// Loads and stores less than 64-bits are already atomic; ones above that
21897// are doomed anyway, so defer to the default libcall and blame the OS when
21898// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21899// anything for those.
21900// FIXME: ldrd and strd are atomic if the CPU has LPAE (e.g. A15 has that
21901// guarantee, see DDI0406C ARM architecture reference manual,
21902// sections A8.8.72-74 LDRD)
21905 bool has64BitLLSC;
21906 if (Subtarget->isMClass())
21907 has64BitLLSC = false;
21908 else if (Subtarget->isThumb())
21909 has64BitLLSC = Subtarget->hasV7Ops();
21910 else
21911 has64BitLLSC = Subtarget->hasV6Ops();
21912
21913 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
21914 if (Size == 64 && has64BitLLSC)
21916
21917 if (Size <= 32 && !hasNativeAtomics(*Subtarget) && !InsertFencesForAtomic &&
21920
21922}
21923
21924// For the real atomic operations, we have ldrex/strex up to 32 bits,
21925// and up to 64 bits on the non-M profiles
21928 if (AI->isFloatingPointOperation())
21930
21931 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
21932 bool hasAtomicRMW;
21933 if (Subtarget->isMClass())
21934 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21935 else if (Subtarget->isThumb())
21936 hasAtomicRMW = Subtarget->hasV7Ops();
21937 else
21938 hasAtomicRMW = Subtarget->hasV6Ops();
21939 if (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW) {
21940 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21941 // implement atomicrmw without spilling. If the target address is also on
21942 // the stack and close enough to the spill slot, this can lead to a
21943 // situation where the monitor always gets cleared and the atomic operation
21944 // can never succeed. So at -O0 lower this operation to a CAS loop.
21945 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
21948 }
21950}
21951
21952// Similar to shouldExpandAtomicRMWInIR, ldrex/strex can be used up to 32
21953// bits, and up to 64 bits on the non-M profiles.
21956 const AtomicCmpXchgInst *AI) const {
21957 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21958 // implement cmpxchg without spilling. If the address being exchanged is also
21959 // on the stack and close enough to the spill slot, this can lead to a
21960 // situation where the monitor always gets cleared and the atomic operation
21961 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
21962 unsigned Size = AI->getOperand(1)->getType()->getPrimitiveSizeInBits();
21963 bool HasAtomicCmpXchg;
21964 if (Subtarget->isMClass())
21965 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21966 else if (Subtarget->isThumb())
21967 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21968 else
21969 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21970 if (getTargetMachine().getOptLevel() != CodeGenOptLevel::None &&
21971 HasAtomicCmpXchg && Size <= (Subtarget->isMClass() ? 32U : 64U))
21974}
21975
21977 const Instruction *I) const {
21978 return InsertFencesForAtomic;
21979}
21980
21982 if (Subtarget->isTargetFuchsia())
21984 // ROPI/RWPI are not supported currently.
21985 return !Subtarget->isROPI() && !Subtarget->isRWPI();
21986}
21987
21988Value *
21990 const LibcallLoweringInfo &Libcalls) const {
21991 // Fuchsia provides a fixed TLS slot for the stack cookie.
21992 // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
21993 if (Subtarget->isTargetFuchsia()) {
21994 Module *M = IRB.getModule();
21995 Function *ThreadPointerFunc = Intrinsic::getOrInsertDeclaration(
21996 M, Intrinsic::thread_pointer, IRB.getPtrTy());
21997 return IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
21998 IRB.CreateCall(ThreadPointerFunc), -0x10);
21999 }
22000
22001 return TargetLowering::getIRStackGuard(IRB, Libcalls);
22002}
22003
22005 Module &M, const LibcallLoweringInfo &Libcalls) const {
22006 // MSVC CRT provides functionalities for stack protection.
22007 RTLIB::LibcallImpl SecurityCheckCookieLibcall =
22008 Libcalls.getLibcallImpl(RTLIB::SECURITY_CHECK_COOKIE);
22009
22010 RTLIB::LibcallImpl SecurityCookieVar =
22011 Libcalls.getLibcallImpl(RTLIB::STACK_CHECK_GUARD);
22012 if (SecurityCheckCookieLibcall != RTLIB::Unsupported &&
22013 SecurityCookieVar != RTLIB::Unsupported) {
22014 // MSVC CRT has a global variable holding security cookie.
22015 const RTLIB::RuntimeLibcallsInfo &RTLCI = Libcalls.getRuntimeLibcallsInfo();
22016 RTLCI.getOrInsertImplGlobal(M, SecurityCookieVar);
22017
22018 // MSVC CRT has a function to validate security cookie.
22019 FunctionCallee SecurityCheckCookie =
22020 RTLCI.getOrInsertLibcallDecl(M, SecurityCheckCookieLibcall);
22021 if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee()))
22022 F->addParamAttr(0, Attribute::AttrKind::InReg);
22023 }
22024
22026}
22027
22029 unsigned &Cost) const {
22030 // If we do not have NEON, vector types are not natively supported.
22031 if (!Subtarget->hasNEON())
22032 return false;
22033
22034 // Floating point values and vector values map to the same register file.
22035 // Therefore, although we could do a store extract of a vector type, this is
22036 // better to leave at float as we have more freedom in the addressing mode for
22037 // those.
22038 if (VectorTy->isFPOrFPVectorTy())
22039 return false;
22040
22041 // If the index is unknown at compile time, this is very expensive to lower
22042 // and it is not possible to combine the store with the extract.
22043 if (!isa<ConstantInt>(Idx))
22044 return false;
22045
22046 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type");
22047 unsigned BitWidth = VectorTy->getPrimitiveSizeInBits().getFixedValue();
22048 // We can do a store + vector extract on any vector that fits perfectly in a D
22049 // or Q register.
22050 if (BitWidth == 64 || BitWidth == 128) {
22051 Cost = 0;
22052 return true;
22053 }
22054 return false;
22055}
22056
22058 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
22059 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
22060 unsigned Opcode = Op.getOpcode();
22061 switch (Opcode) {
22062 case ARMISD::VORRIMM:
22063 case ARMISD::VBICIMM:
22064 return false;
22065 }
22067 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
22068}
22069
22071 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
22072}
22073
22075 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
22076}
22077
22079 const Instruction &AndI) const {
22080 if (!Subtarget->hasV7Ops())
22081 return false;
22082
22083 // Sink the `and` instruction only if the mask would fit into a modified
22084 // immediate operand.
22086 if (!Mask || Mask->getValue().getBitWidth() > 32u)
22087 return false;
22088 auto MaskVal = unsigned(Mask->getValue().getZExtValue());
22089 return (Subtarget->isThumb2() ? ARM_AM::getT2SOImmVal(MaskVal)
22090 : ARM_AM::getSOImmVal(MaskVal)) != -1;
22091}
22092
22095 SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
22096 if (Subtarget->hasMinSize() && !getTM().getTargetTriple().isOSWindows())
22099 ExpansionFactor);
22100}
22101
22103 Value *Addr,
22104 AtomicOrdering Ord) const {
22105 Module *M = Builder.getModule();
22106 bool IsAcquire = isAcquireOrStronger(Ord);
22107
22108 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd
22109 // intrinsic must return {i32, i32} and we have to recombine them into a
22110 // single i64 here.
22111 if (ValueTy->getPrimitiveSizeInBits() == 64) {
22113 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
22114
22115 Value *LoHi =
22116 Builder.CreateIntrinsic(Int, Addr, /*FMFSource=*/nullptr, "lohi");
22117
22118 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
22119 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
22120 if (!Subtarget->isLittle())
22121 std::swap (Lo, Hi);
22122 Lo = Builder.CreateZExt(Lo, ValueTy, "lo64");
22123 Hi = Builder.CreateZExt(Hi, ValueTy, "hi64");
22124 return Builder.CreateOr(
22125 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValueTy, 32)), "val64");
22126 }
22127
22128 Type *Tys[] = { Addr->getType() };
22129 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
22130 CallInst *CI = Builder.CreateIntrinsicWithoutFolding(Int, Tys, Addr);
22131
22132 CI->addParamAttr(
22133 0, Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
22134 return Builder.CreateTruncOrBitCast(CI, ValueTy);
22135}
22136
22138 IRBuilderBase &Builder) const {
22139 if (!(Subtarget->isThumb() ? Subtarget->hasV7Clrex()
22140 : Subtarget->hasV6KOps()))
22141 return;
22142 Builder.CreateIntrinsic(Intrinsic::arm_clrex, {});
22143}
22144
22146 Value *Val, Value *Addr,
22147 AtomicOrdering Ord) const {
22148 Module *M = Builder.getModule();
22149 bool IsRelease = isReleaseOrStronger(Ord);
22150
22151 // Since the intrinsics must have legal type, the i64 intrinsics take two
22152 // parameters: "i32, i32". We must marshal Val into the appropriate form
22153 // before the call.
22154 if (Val->getType()->getPrimitiveSizeInBits() == 64) {
22156 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
22157 Type *Int32Ty = Type::getInt32Ty(M->getContext());
22158
22159 Value *Lo = Builder.CreateTrunc(Val, Int32Ty, "lo");
22160 Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32), Int32Ty, "hi");
22161 if (!Subtarget->isLittle())
22162 std::swap(Lo, Hi);
22163 return Builder.CreateIntrinsic(Int, {Lo, Hi, Addr});
22164 }
22165
22166 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
22167 Type *Tys[] = { Addr->getType() };
22169
22170 CallInst *CI = Builder.CreateCall(
22171 Strex, {Builder.CreateZExtOrBitCast(
22172 Val, Strex->getFunctionType()->getParamType(0)),
22173 Addr});
22174 CI->addParamAttr(1, Attribute::get(M->getContext(), Attribute::ElementType,
22175 Val->getType()));
22176 return CI;
22177}
22178
22179
22181 return Subtarget->isMClass();
22182}
22183
22184/// A helper function for determining the number of interleaved accesses we
22185/// will generate when lowering accesses of the given type.
22186unsigned
22188 const DataLayout &DL) const {
22189 return (DL.getTypeSizeInBits(VecTy) + 127) / 128;
22190}
22191
22193 unsigned Factor, FixedVectorType *VecTy, Align Alignment,
22194 const DataLayout &DL) const {
22195
22196 unsigned VecSize = DL.getTypeSizeInBits(VecTy);
22197 unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
22198
22199 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
22200 return false;
22201
22202 // Ensure the vector doesn't have f16 elements. Even though we could do an
22203 // i16 vldN, we can't hold the f16 vectors and will end up converting via
22204 // f32.
22205 if (Subtarget->hasNEON() && VecTy->getElementType()->isHalfTy())
22206 return false;
22207 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
22208 return false;
22209
22210 // Ensure the number of vector elements is greater than 1.
22211 if (VecTy->getNumElements() < 2)
22212 return false;
22213
22214 // Ensure the element type is legal.
22215 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
22216 return false;
22217 // And the alignment if high enough under MVE.
22218 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
22219 return false;
22220
22221 // Ensure the total vector size is 64 or a multiple of 128. Types larger than
22222 // 128 will be split into multiple interleaved accesses.
22223 if (Subtarget->hasNEON() && VecSize == 64)
22224 return true;
22225 return VecSize % 128 == 0;
22226}
22227
22229 if (Subtarget->hasNEON())
22230 return 4;
22231 if (Subtarget->hasMVEIntegerOps())
22234}
22235
22236/// Lower an interleaved load into a vldN intrinsic.
22237///
22238/// E.g. Lower an interleaved load (Factor = 2):
22239/// %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
22240/// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
22241/// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
22242///
22243/// Into:
22244/// %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4)
22245/// %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0
22246/// %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
22249 ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const {
22250 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
22251 "Invalid interleave factor");
22252 assert(!Shuffles.empty() && "Empty shufflevector input");
22253 assert(Shuffles.size() == Indices.size() &&
22254 "Unmatched number of shufflevectors and indices");
22255
22256 auto *LI = dyn_cast<LoadInst>(Load);
22257 if (!LI)
22258 return false;
22259 assert(!Mask && GapMask.popcount() == Factor && "Unexpected mask on a load");
22260
22261 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->getType());
22262 Type *EltTy = VecTy->getElementType();
22263
22264 const DataLayout &DL = LI->getDataLayout();
22265 Align Alignment = LI->getAlign();
22266
22267 // Skip if we do not have NEON and skip illegal vector types. We can
22268 // "legalize" wide vector types into multiple interleaved accesses as long as
22269 // the vector types are divisible by 128.
22270 if (!isLegalInterleavedAccessType(Factor, VecTy, Alignment, DL))
22271 return false;
22272
22273 unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL);
22274
22275 // A pointer vector can not be the return type of the ldN intrinsics. Need to
22276 // load integer vectors first and then convert to pointer vectors.
22277 if (EltTy->isPointerTy())
22278 VecTy = FixedVectorType::get(DL.getIntPtrType(EltTy), VecTy);
22279
22280 IRBuilder<> Builder(LI);
22281 Builder.disableGEPCanonicalization();
22282
22283 // The base address of the load.
22284 Value *BaseAddr = LI->getPointerOperand();
22285
22286 if (NumLoads > 1) {
22287 // If we're going to generate more than one load, reset the sub-vector type
22288 // to something legal.
22289 VecTy = FixedVectorType::get(VecTy->getElementType(),
22290 VecTy->getNumElements() / NumLoads);
22291 }
22292
22293 assert(isTypeLegal(EVT::getEVT(VecTy)) && "Illegal vldN vector type!");
22294
22295 auto createLoadIntrinsic = [&](Value *BaseAddr) {
22296 if (Subtarget->hasNEON()) {
22297 Type *PtrTy = Builder.getPtrTy(LI->getPointerAddressSpace());
22298 Type *Tys[] = {VecTy, PtrTy};
22299 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
22300 Intrinsic::arm_neon_vld3,
22301 Intrinsic::arm_neon_vld4};
22302
22304 Ops.push_back(BaseAddr);
22305 Ops.push_back(Builder.getInt32(LI->getAlign().value()));
22306
22307 return Builder.CreateIntrinsic(LoadInts[Factor - 2], Tys, Ops,
22308 /*FMFSource=*/nullptr, "vldN");
22309 } else {
22310 assert((Factor == 2 || Factor == 4) &&
22311 "expected interleave factor of 2 or 4 for MVE");
22312 Intrinsic::ID LoadInts =
22313 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
22314 Type *PtrTy = Builder.getPtrTy(LI->getPointerAddressSpace());
22315 Type *Tys[] = {VecTy, PtrTy};
22316
22318 Ops.push_back(BaseAddr);
22319 return Builder.CreateIntrinsic(LoadInts, Tys, Ops, /*FMFSource=*/nullptr,
22320 "vldN");
22321 }
22322 };
22323
22324 // Holds sub-vectors extracted from the load intrinsic return values. The
22325 // sub-vectors are associated with the shufflevector instructions they will
22326 // replace.
22328
22329 for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
22330 // If we're generating more than one load, compute the base address of
22331 // subsequent loads as an offset from the previous.
22332 if (LoadCount > 0)
22333 BaseAddr = Builder.CreateConstGEP1_32(VecTy->getElementType(), BaseAddr,
22334 VecTy->getNumElements() * Factor);
22335
22336 Value *VldN = createLoadIntrinsic(BaseAddr);
22337
22338 // Replace uses of each shufflevector with the corresponding vector loaded
22339 // by ldN.
22340 for (unsigned i = 0; i < Shuffles.size(); i++) {
22341 ShuffleVectorInst *SV = Shuffles[i];
22342 unsigned Index = Indices[i];
22343
22344 Value *SubVec = Builder.CreateExtractValue(VldN, Index);
22345
22346 // Convert the integer vector to pointer vector if the element is pointer.
22347 if (EltTy->isPointerTy())
22348 SubVec = Builder.CreateIntToPtr(
22349 SubVec,
22350 FixedVectorType::get(SV->getType()->getElementType(), VecTy));
22351
22352 SubVecs[SV].push_back(SubVec);
22353 }
22354 }
22355
22356 // Replace uses of the shufflevector instructions with the sub-vectors
22357 // returned by the load intrinsic. If a shufflevector instruction is
22358 // associated with more than one sub-vector, those sub-vectors will be
22359 // concatenated into a single wide vector.
22360 for (ShuffleVectorInst *SVI : Shuffles) {
22361 auto &SubVec = SubVecs[SVI];
22362 auto *WideVec =
22363 SubVec.size() > 1 ? concatenateVectors(Builder, SubVec) : SubVec[0];
22364 SVI->replaceAllUsesWith(WideVec);
22365 }
22366
22367 return true;
22368}
22369
22370/// Lower an interleaved store into a vstN intrinsic.
22371///
22372/// E.g. Lower an interleaved store (Factor = 3):
22373/// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
22374/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
22375/// store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4
22376///
22377/// Into:
22378/// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
22379/// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
22380/// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
22381/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
22382///
22383/// Note that the new shufflevectors will be removed and we'll only generate one
22384/// vst3 instruction in CodeGen.
22385///
22386/// Example for a more general valid mask (Factor 3). Lower:
22387/// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1,
22388/// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
22389/// store <12 x i32> %i.vec, <12 x i32>* %ptr
22390///
22391/// Into:
22392/// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7>
22393/// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35>
22394/// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19>
22395/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
22397 Value *LaneMask,
22398 ShuffleVectorInst *SVI,
22399 unsigned Factor,
22400 const APInt &GapMask) const {
22401 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
22402 "Invalid interleave factor");
22403 auto *SI = dyn_cast<StoreInst>(Store);
22404 if (!SI)
22405 return false;
22406 assert(!LaneMask && GapMask.popcount() == Factor &&
22407 "Unexpected mask on store");
22408
22409 auto *VecTy = cast<FixedVectorType>(SVI->getType());
22410 assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store");
22411
22412 unsigned LaneLen = VecTy->getNumElements() / Factor;
22413 Type *EltTy = VecTy->getElementType();
22414 auto *SubVecTy = FixedVectorType::get(EltTy, LaneLen);
22415
22416 const DataLayout &DL = SI->getDataLayout();
22417 Align Alignment = SI->getAlign();
22418
22419 // Skip if we do not have NEON and skip illegal vector types. We can
22420 // "legalize" wide vector types into multiple interleaved accesses as long as
22421 // the vector types are divisible by 128.
22422 if (!isLegalInterleavedAccessType(Factor, SubVecTy, Alignment, DL))
22423 return false;
22424
22425 unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL);
22426
22427 Value *Op0 = SVI->getOperand(0);
22428 Value *Op1 = SVI->getOperand(1);
22429 IRBuilder<> Builder(SI);
22430 Builder.disableGEPCanonicalization();
22431
22432 // StN intrinsics don't support pointer vectors as arguments. Convert pointer
22433 // vectors to integer vectors.
22434 if (EltTy->isPointerTy()) {
22435 Type *IntTy = DL.getIntPtrType(EltTy);
22436
22437 // Convert to the corresponding integer vector.
22438 auto *IntVecTy =
22440 Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
22441 Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
22442
22443 SubVecTy = FixedVectorType::get(IntTy, LaneLen);
22444 }
22445
22446 // The base address of the store.
22447 Value *BaseAddr = SI->getPointerOperand();
22448
22449 if (NumStores > 1) {
22450 // If we're going to generate more than one store, reset the lane length
22451 // and sub-vector type to something legal.
22452 LaneLen /= NumStores;
22453 SubVecTy = FixedVectorType::get(SubVecTy->getElementType(), LaneLen);
22454 }
22455
22456 assert(isTypeLegal(EVT::getEVT(SubVecTy)) && "Illegal vstN vector type!");
22457
22458 auto Mask = SVI->getShuffleMask();
22459
22460 auto createStoreIntrinsic = [&](Value *BaseAddr,
22461 SmallVectorImpl<Value *> &Shuffles) {
22462 if (Subtarget->hasNEON()) {
22463 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
22464 Intrinsic::arm_neon_vst3,
22465 Intrinsic::arm_neon_vst4};
22466 Type *PtrTy = Builder.getPtrTy(SI->getPointerAddressSpace());
22467 Type *Tys[] = {PtrTy, SubVecTy};
22468
22470 Ops.push_back(BaseAddr);
22471 append_range(Ops, Shuffles);
22472 Ops.push_back(Builder.getInt32(SI->getAlign().value()));
22473 Builder.CreateIntrinsic(StoreInts[Factor - 2], Tys, Ops);
22474 } else {
22475 assert((Factor == 2 || Factor == 4) &&
22476 "expected interleave factor of 2 or 4 for MVE");
22477 Intrinsic::ID StoreInts =
22478 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
22479 Type *PtrTy = Builder.getPtrTy(SI->getPointerAddressSpace());
22480 Type *Tys[] = {PtrTy, SubVecTy};
22481
22483 Ops.push_back(BaseAddr);
22484 append_range(Ops, Shuffles);
22485 for (unsigned F = 0; F < Factor; F++) {
22486 Ops.push_back(Builder.getInt32(F));
22487 Builder.CreateIntrinsic(StoreInts, Tys, Ops);
22488 Ops.pop_back();
22489 }
22490 }
22491 };
22492
22493 for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
22494 // If we generating more than one store, we compute the base address of
22495 // subsequent stores as an offset from the previous.
22496 if (StoreCount > 0)
22497 BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getElementType(),
22498 BaseAddr, LaneLen * Factor);
22499
22500 SmallVector<Value *, 4> Shuffles;
22501
22502 // Split the shufflevector operands into sub vectors for the new vstN call.
22503 for (unsigned i = 0; i < Factor; i++) {
22504 unsigned IdxI = StoreCount * LaneLen * Factor + i;
22505 if (Mask[IdxI] >= 0) {
22506 Shuffles.push_back(Builder.CreateShuffleVector(
22507 Op0, Op1, createSequentialMask(Mask[IdxI], LaneLen, 0)));
22508 } else {
22509 unsigned StartMask = 0;
22510 for (unsigned j = 1; j < LaneLen; j++) {
22511 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
22512 if (Mask[IdxJ * Factor + IdxI] >= 0) {
22513 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
22514 break;
22515 }
22516 }
22517 // Note: If all elements in a chunk are undefs, StartMask=0!
22518 // Note: Filling undef gaps with random elements is ok, since
22519 // those elements were being written anyway (with undefs).
22520 // In the case of all undefs we're defaulting to using elems from 0
22521 // Note: StartMask cannot be negative, it's checked in
22522 // isReInterleaveMask
22523 Shuffles.push_back(Builder.CreateShuffleVector(
22524 Op0, Op1, createSequentialMask(StartMask, LaneLen, 0)));
22525 }
22526 }
22527
22528 createStoreIntrinsic(BaseAddr, Shuffles);
22529 }
22530 return true;
22531}
22532
22540
22542 uint64_t &Members) {
22543 if (auto *ST = dyn_cast<StructType>(Ty)) {
22544 for (unsigned i = 0; i < ST->getNumElements(); ++i) {
22545 uint64_t SubMembers = 0;
22546 if (!isHomogeneousAggregate(ST->getElementType(i), Base, SubMembers))
22547 return false;
22548 Members += SubMembers;
22549 }
22550 } else if (auto *AT = dyn_cast<ArrayType>(Ty)) {
22551 uint64_t SubMembers = 0;
22552 if (!isHomogeneousAggregate(AT->getElementType(), Base, SubMembers))
22553 return false;
22554 Members += SubMembers * AT->getNumElements();
22555 } else if (Ty->isFloatTy()) {
22556 if (Base != HA_UNKNOWN && Base != HA_FLOAT)
22557 return false;
22558 Members = 1;
22559 Base = HA_FLOAT;
22560 } else if (Ty->isDoubleTy()) {
22561 if (Base != HA_UNKNOWN && Base != HA_DOUBLE)
22562 return false;
22563 Members = 1;
22564 Base = HA_DOUBLE;
22565 } else if (auto *VT = dyn_cast<VectorType>(Ty)) {
22566 Members = 1;
22567 switch (Base) {
22568 case HA_FLOAT:
22569 case HA_DOUBLE:
22570 return false;
22571 case HA_VECT64:
22572 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
22573 case HA_VECT128:
22574 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
22575 case HA_UNKNOWN:
22576 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
22577 case 64:
22578 Base = HA_VECT64;
22579 return true;
22580 case 128:
22581 Base = HA_VECT128;
22582 return true;
22583 default:
22584 return false;
22585 }
22586 }
22587 }
22588
22589 return (Members > 0 && Members <= 4);
22590}
22591
22592/// Return the correct alignment for the current calling convention.
22594 Type *ArgTy, const DataLayout &DL) const {
22595 const Align ABITypeAlign = DL.getABITypeAlign(ArgTy);
22596 if (!ArgTy->isVectorTy())
22597 return ABITypeAlign;
22598
22599 // Avoid over-aligning vector parameters. It would require realigning the
22600 // stack and waste space for no real benefit.
22601 MaybeAlign StackAlign = DL.getStackAlignment();
22602 assert(StackAlign && "data layout string is missing stack alignment");
22603 return std::min(ABITypeAlign, *StackAlign);
22604}
22605
22606/// Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
22607/// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
22608/// passing according to AAPCS rules.
22610 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
22611 const DataLayout &DL) const {
22612 if (getEffectiveCallingConv(CallConv, isVarArg) !=
22614 return false;
22615
22617 uint64_t Members = 0;
22618 bool IsHA = isHomogeneousAggregate(Ty, Base, Members);
22619 LLVM_DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump());
22620
22621 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
22622 return IsHA || IsIntArray;
22623}
22624
22626 ExceptionHandling EH, const Constant *PersonalityFn) const {
22627 // Platforms which do not use SjLj EH may return values in these registers
22628 // via the personality function.
22629 return EH == ExceptionHandling::SjLj ? Register() : ARM::R0;
22630}
22631
22633 ExceptionHandling EH, const Constant *PersonalityFn) const {
22634 // Platforms which do not use SjLj EH may return values in these registers
22635 // via the personality function.
22636 return EH == ExceptionHandling::SjLj ? Register() : ARM::R1;
22637}
22638
22639void ARMTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
22640 // Update IsSplitCSR in ARMFunctionInfo.
22641 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>();
22642 AFI->setIsSplitCSR(true);
22643}
22644
22645void ARMTargetLowering::insertCopiesSplitCSR(
22646 MachineBasicBlock *Entry,
22647 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
22648 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
22649 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
22650 if (!IStart)
22651 return;
22652
22653 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
22654 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
22655 MachineBasicBlock::iterator MBBI = Entry->begin();
22656 for (const MCPhysReg *I = IStart; *I; ++I) {
22657 const TargetRegisterClass *RC = nullptr;
22658 if (ARM::GPRRegClass.contains(*I))
22659 RC = &ARM::GPRRegClass;
22660 else if (ARM::DPRRegClass.contains(*I))
22661 RC = &ARM::DPRRegClass;
22662 else
22663 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
22664
22665 Register NewVR = MRI->createVirtualRegister(RC);
22666 // Create copy from CSR to a virtual register.
22667 // FIXME: this currently does not emit CFI pseudo-instructions, it works
22668 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
22669 // nounwind. If we want to generalize this later, we may need to emit
22670 // CFI pseudo-instructions.
22671 assert(Entry->getParent()->getFunction().hasFnAttribute(
22672 Attribute::NoUnwind) &&
22673 "Function should be nounwind in insertCopiesSplitCSR!");
22674 Entry->addLiveIn(*I);
22675 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
22676 .addReg(*I);
22677
22678 // Insert the copy-back instructions right before the terminator.
22679 for (auto *Exit : Exits)
22680 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
22681 TII->get(TargetOpcode::COPY), *I)
22682 .addReg(NewVR);
22683 }
22684}
22685
22690
22692 return Subtarget->hasMVEIntegerOps();
22693}
22694
22696 ComplexDeinterleavingOperation Operation, Type *Ty) const {
22697 auto *VTy = dyn_cast<FixedVectorType>(Ty);
22698 if (!VTy)
22699 return false;
22700
22701 auto *ScalarTy = VTy->getScalarType();
22702 unsigned NumElements = VTy->getNumElements();
22703
22704 unsigned VTyWidth = VTy->getScalarSizeInBits() * NumElements;
22705 if (VTyWidth < 128 || !llvm::isPowerOf2_32(VTyWidth))
22706 return false;
22707
22708 // Both VCADD and VCMUL/VCMLA support the same types, F16 and F32
22709 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22710 return Subtarget->hasMVEFloatOps();
22711
22712 if (Operation != ComplexDeinterleavingOperation::CAdd)
22713 return false;
22714
22715 return Subtarget->hasMVEIntegerOps() &&
22716 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22717 ScalarTy->isIntegerTy(32));
22718}
22719
22721 static const MCPhysReg RCRegs[] = {ARM::FPSCR_RM};
22722 return RCRegs;
22723}
22724
22727 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
22728 Value *Accumulator) const {
22729
22731
22732 unsigned TyWidth = Ty->getScalarSizeInBits() * Ty->getNumElements();
22733
22734 assert(TyWidth >= 128 && "Width of vector type must be at least 128 bits");
22735
22736 if (TyWidth > 128) {
22737 int Stride = Ty->getNumElements() / 2;
22738 auto SplitSeq = llvm::seq<int>(0, Ty->getNumElements());
22739 auto SplitSeqVec = llvm::to_vector(SplitSeq);
22740 ArrayRef<int> LowerSplitMask(&SplitSeqVec[0], Stride);
22741 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22742
22743 auto *LowerSplitA = B.CreateShuffleVector(InputA, LowerSplitMask);
22744 auto *LowerSplitB = B.CreateShuffleVector(InputB, LowerSplitMask);
22745 auto *UpperSplitA = B.CreateShuffleVector(InputA, UpperSplitMask);
22746 auto *UpperSplitB = B.CreateShuffleVector(InputB, UpperSplitMask);
22747 Value *LowerSplitAcc = nullptr;
22748 Value *UpperSplitAcc = nullptr;
22749
22750 if (Accumulator) {
22751 LowerSplitAcc = B.CreateShuffleVector(Accumulator, LowerSplitMask);
22752 UpperSplitAcc = B.CreateShuffleVector(Accumulator, UpperSplitMask);
22753 }
22754
22755 auto *LowerSplitInt = createComplexDeinterleavingIR(
22756 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22757 auto *UpperSplitInt = createComplexDeinterleavingIR(
22758 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22759
22760 ArrayRef<int> JoinMask(&SplitSeqVec[0], Ty->getNumElements());
22761 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22762 }
22763
22764 auto *IntTy = Type::getInt32Ty(B.getContext());
22765
22766 ConstantInt *ConstRotation = nullptr;
22767 if (OperationType == ComplexDeinterleavingOperation::CMulPartial) {
22768 ConstRotation = ConstantInt::get(IntTy, (int)Rotation);
22769
22770 if (Accumulator)
22771 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22772 {ConstRotation, Accumulator, InputB, InputA});
22773 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22774 {ConstRotation, InputB, InputA});
22775 }
22776
22777 if (OperationType == ComplexDeinterleavingOperation::CAdd) {
22778 // 1 means the value is not halved.
22779 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22780
22782 ConstRotation = ConstantInt::get(IntTy, 0);
22784 ConstRotation = ConstantInt::get(IntTy, 1);
22785
22786 if (!ConstRotation)
22787 return nullptr; // Invalid rotation for arm_mve_vcaddq
22788
22789 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22790 {ConstHalving, ConstRotation, InputA, InputB});
22791 }
22792
22793 return nullptr;
22794}
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
auto * MemN
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static SDValue carryFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG, bool Invert)
static SDValue overflowFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
unsigned NumParts
return SDValue()
static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG)
static const MCPhysReg GPRArgRegs[]
static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert)
constexpr MVT FlagsVT
Value type used for NZCV flags.
static unsigned getCmpOperandFoldingProfit(SDValue Op, bool AllowExtend)
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm, const APInt &Demanded, TargetLowering::TargetLoweringOpt &TLO, unsigned NewOpc)
static bool isSafeSignedCMN(SDValue Op, SelectionDAG &DAG)
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
SDValue Pred
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
uint64_t uint64_t
uint64_t Imm
static bool isConstant(const MachineInstr &MI)
constexpr LLT F64
constexpr LLT S1
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isIncompatibleReg(const MCPhysReg &PR, MVT VT)
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVECTOR_DEINTERLEAVECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static SDValue PerformLegalizedVECTOR_DEINTERLEAVECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations...
static SDValue createGPRPairNode2xi32(SelectionDAG &DAG, SDValue V0, SDValue V1)
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static SDValue PerformVMOVrsCombine(SDNode *N, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg, SelectionDAG &DAG, const SDLoc &DL)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static SDValue createGPRPairNodei64(SelectionDAG &DAG, SDValue V)
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static SDValue performNegCMovCombine(SDNode *N, SelectionDAG &DAG)
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, FunctionType *FuncTy, const AttributeList &FuncAttrs, RTLIB::LibcallImpl LCImpl)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformInterleavedStoreCombine(StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static SDValue matchCSET(unsigned &Opcode, bool &InvertCond, SDValue TrueVal, SDValue FalseVal, const ARMSubtarget *Subtarget)
static bool isSequentialConcatOfVectorInterleave(SDNode *N, SmallVectorImpl< SDValue > &Ops)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getInvertedARMCondCode(SDValue ARMcc, SelectionDAG &DAG)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool hasNativeAtomics(const ARMSubtarget &ST)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
cl::opt< unsigned > ArmMaxBaseUpdatesToCheck("arm-max-base-updates-to-check", cl::Hidden, cl::desc("Maximum number of base-updates to check generating postindex."), cl::init(64))
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG, unsigned Opcode, bool IsSigned)
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static int getNegationCost(SDValue Op)
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static bool isLegalLogicalImmediate(unsigned Imm, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static SDValue PerformLegalizedInterleavedStoreCombine(StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformORCombineToShiftInsert(SelectionDAG &DAG, SDValue AndOp, SDValue ShiftOp, EVT VT, SDLoc dl)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue getShiftedLowMaskZeroTest(SDValue Value, SelectionDAG &DAG, const SDLoc &DL)
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
arm ldst static false bool definesCPSR(const MachineInstr &MI)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
This file a TargetTransformInfoImplBase conforming object specific to the ARM target machine.
Function Alias Analysis false
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
This file implements the BitVector class.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static void createLoadIntrinsic(IntrinsicInst *II, LoadInst *LI, dxil::ResourceTypeInfo &RTI)
static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI, dxil::ResourceTypeInfo &RTI)
This file defines the DenseMap class.
static bool isSigned(unsigned Opcode)
#define Check(C,...)
#define op(i)
#define im(i)
const HexagonInstrInfo * TII
hexagon merge activate weight
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
static LVOptions Options
Definition LVOptions.cpp:25
lazy value info
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define G(x, y, z)
Definition MD5.cpp:55
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define T
nvptx lower args
uint64_t High
uint64_t IntrinsicInst * II
R600 Clause Merge
SmallVector< MachineOperand, 4 > Cond
int64_t C1
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
int64_t C0
MachineInstr *& Lo
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Contains matchers for matching SelectionDAG nodes and values.
SI Lower i1 Copies
Func MI getDebugLoc()))
This file contains some templates that are useful if you are working with the STL at all.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
This file contains some functions that are useful when dealing with strings.
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This file describes how to lower LLVM code to machine code.
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
Value * RHS
Value * LHS
BinaryOperator * Mul
static bool isIntrinsic(const CallBase &Call, Intrinsic::ID ID)
The Input class is used to parse a yaml document into in-memory structs and vectors.
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
LLVM_ABI bool getExactInverse(APFloat *Inv) const
If this value is normal and has an exact, normal, multiplicative inverse, store it in inv and return ...
Definition APFloat.cpp:5987
APInt bitcastToAPInt() const
Definition APFloat.h:1475
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition APFloat.h:1436
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition APInt.h:419
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned popcount() const
Count the number of bits set.
Definition APInt.h:1690
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition APInt.cpp:1078
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition APInt.h:1350
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition APInt.h:1205
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1115
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition APInt.h:1618
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition APInt.cpp:648
unsigned logBase2() const
Definition APInt.h:1781
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition APInt.h:471
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition APInt.h:1261
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:292
bool isOne() const
Determine if this is a value of 1.
Definition APInt.h:385
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition APInt.h:860
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:853
unsigned countr_one() const
Count the number of trailing one bits.
Definition APInt.h:1676
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1225
An arbitrary precision integer that knows its signedness.
Definition APSInt.h:24
const ARMBaseRegisterInfo & getRegisterInfo() const
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj, ARMCP::ARMCPModifier Modifier=ARMCP::no_modifier, bool AddCurrentAddress=false)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
SmallVectorImpl< ForwardedRegister > & getForwardedMustTailRegParms()
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
void setArgRegsSaveSize(unsigned s)
void setReturnRegsCount(unsigned s)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X, SDValue Y) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
FPImmAction getFPImmAction(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
Returns how to materialize the specified FP immediate.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo, const LibcallLoweringInfo *libcallLowering) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void insertSSPDeclarations(Module &M, const LibcallLoweringInfo &Libcalls) const override
Inserts necessary declarations for SSP (stack protection) purpose.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool lowerInterleavedStore(Instruction *Store, Value *Mask, ShuffleVectorInst *SVI, unsigned Factor, const APInt &GapMask) const override
Lower an interleaved store into a vstN intrinsic.
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
bool useLoadStackGuardNode(const Module &M) const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
bool lowerInterleavedLoad(Instruction *Load, Value *Mask, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor, const APInt &GapMask) const override
Lower an interleaved load into a vldN intrinsic.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool preferSelectsOverBooleanArithmetic(EVT VT) const override
Should we prefer selects to doing arithmetic on boolean types.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return the cost of EXTRACT_SUBVECTOR for this result type with this index.
CallingConv::ID getEffectiveCallingConv(CallingConv::ID CC, bool isVarArg) const
getEffectiveCallingConv - Get the effective calling convention, taking into account presence of float...
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
bool supportKCFIBundles() const override
Return true if the target supports kcfi operand bundles.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
Value * getIRStackGuard(IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const override
If the target has a standard location for the stack protector guard, returns the address of that loca...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy,Idx).
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
bool hasAndNotCompare(SDValue V) const override
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT.
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
const ARMBaseTargetMachine & getTM() const
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static LLVM_ABI BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, and fixed/scalable offsets.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
The address of a basic block.
Definition Constants.h:1088
static constexpr BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
LLVM_ABI int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
LLVM_ABI void analyzeMustTailForwardedRegisters(SmallVectorImpl< ForwardedRegister > &Forwards, ArrayRef< MVT > RegParmTypes, CCAssignFn Fn)
Compute the set of registers that need to be preserved and forwarded to any musttail calls.
static LLVM_ABI bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
LLVM_ABI bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
LLVM_ABI void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void addInRegsParamInfo(unsigned RegBegin, unsigned RegEnd)
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
bool needsCustom() const
int64_t getLocMemOffset() const
unsigned getValNo() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
AttributeList getAttributes() const
Return the attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
Definition Constants.h:878
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
This is the shared class of boolean and integer constants.
Definition Constants.h:87
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
MachineConstantPoolValue * getMachineCPVal() const
const Constant * getConstVal() const
LLVM_ABI Type * getType() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
bool isLittleEndian() const
Layout endianness...
Definition DataLayout.h:220
bool isBigEndian() const
Definition DataLayout.h:221
MaybeAlign getStackAlignment() const
Returns the natural stack alignment, or MaybeAlign() if one wasn't specified.
Definition DataLayout.h:253
LLVM_ABI TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
StringRef getInternalSymbolPrefix() const
Definition DataLayout.h:311
LLVM_ABI Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
A debug info location.
Definition DebugLoc.h:126
bool empty() const
Definition DenseMap.h:716
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
unsigned size() const
Definition DenseMap.h:717
iterator begin()
Definition DenseMap.h:682
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition FastISel.h:67
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:705
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
arg_iterator arg_begin()
Definition Function.h:862
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
Definition Function.h:673
const Argument * const_arg_iterator
Definition Function.h:74
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition Function.h:230
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
const GlobalValue * getGlobal() const
bool isDSOLocal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Definition GlobalValue.h:60
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Definition IRBuilder.h:2026
Module * getModule() const
Get the module.
Definition IRBuilder.h:190
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:2640
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:571
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition IRBuilder.h:518
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
LLVM_ABI bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
Tracks which library functions to use for a particular subtarget or function.
const RTLIB::RuntimeLibcallsInfo & getRuntimeLibcallsInfo() const
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
An instruction for reading from memory.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
Machine Value Type.
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static LLVM_ABI MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
bool is64BitVector() const
Return true if this is a 64-bit vector type.
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
LLVM_ABI MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
LLVM_ABI MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
iterator_range< pred_iterator > predecessors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
LLVM_ABI void moveAfter(MachineBasicBlock *NewBefore)
LLVM_ABI bool isLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
LLVM_ABI unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
LLVM_ABI void computeMaxCallFrameSize(MachineFunction &MF, std::vector< MachineBasicBlock::iterator > *FrameSDOps=nullptr)
Computes the maximum size of a callframe.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
bool hasMustTailInVarArgFunc() const
Returns true if the function is variadic and contains a musttail call.
void setReturnAddressIsTaken(bool s)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
MachineOperand * mop_iterator
iterator/begin/end - Iterate over all operands of a machine instruction.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
LLVM_ABI void setIsRenamable(bool Val=true)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setIsDead(bool Val=true)
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
LLVM_ABI void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
Align getAlign() const
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition Module.h:328
Wrapper class representing virtual and physical registers.
Definition Register.h:20
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
LLVM_ABI bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
ISD::CondCode getOperandCondCode(unsigned Num) const
Helper method returns the CondCode of a CondCodeSDNode.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
LLVM_ABI bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
iterator_range< user_iterator > users()
void setFlags(SDNodeFlags NewFlags)
user_iterator user_begin() const
Provide iteration support to walk over all users of an SDNode.
op_iterator op_end() const
op_iterator op_begin() const
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags, bool AllowCommute=false)
Get the specified node if it's already available, or else return NULL.
LLVM_ABI SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
LLVM_ABI bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, TypeSize Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI bool areNonVolatileConsecutiveStores(StoreSDNode *ST, StoreSDNode *Base, TypeSize Bytes, int Dist) const
Return true if stores are next to each other and can be merged.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
LLVM_ABI MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the signed addition of 2 nodes can overflow.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
void addCallSiteInfo(const SDNode *Node, CallSiteInfo &&CallInfo)
Set CallSiteInfo to be associated with Node.
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
LLVM_ABI SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static LLVM_ABI void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static LLVM_ABI bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
static LLVM_ABI bool isSplatMask(ArrayRef< int > Mask)
void insert_range(Range &&R)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
bool empty() const
Definition SmallSet.h:169
bool erase(const T &V)
Definition SmallSet.h:200
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void resize(size_type N)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
const unsigned char * bytes_end() const
Definition StringRef.h:125
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
const unsigned char * bytes_begin() const
Definition StringRef.h:122
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
virtual Value * getIRStackGuard(IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
const TargetMachine & getTargetMachine() const
virtual void insertSSPDeclarations(Module &M, const LibcallLoweringInfo &Libcalls) const
Inserts necessary declarations for SSP (stack protection) purpose.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const
Returns true if the specified FP immediate can be materialized without a load from the constant pool.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
ExtractSubvectorCost
Enum that specifies how expensive lowering an EXTRACT_SUBVECTOR is.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
FPImmAction
How the legalizer should materialize a floating-point immediate.
@ ConstantPool
Load the immediate from the constant pool.
@ Legal
The target can instruction select the immediate natively.
@ Bitcast
Build the immediate as a bitcast of an integer constant with the same bits.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Get the libcall impl routine name for the specified libcall.
static StringRef getLibcallImplName(RTLIB::LibcallImpl Call)
Get the libcall routine name for the specified libcall implementation.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using an n/2-bit algorithm.
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
static ArgListTy getArgListForFunctionType(FunctionType *FuncTy, const AttributeList &FuncAttrs, ArrayRef< SDValue > Ops)
Build a call argument list for FuncTy, taking the argument node values from Ops and the parameter typ...
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
TargetLowering(const TargetLowering &)=delete
virtual ArrayRef< MCPhysReg > getRoundingControlRegisters() const
Returns a 0 terminated array of rounding control registers that can be attached into strict FP call.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
void setTypeIdForCallsiteInfo(const CallBase *CB, MachineFunction &MF, MachineFunction::CallSiteInfo &CSInfo) const
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
TargetOptions Options
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:293
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition Type.h:144
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
Definition Use.cpp:35
User * getUser() const
Returns the User that contains this Use.
Definition Use.h:61
Value * getOperand(unsigned i) const
Definition User.h:207
unsigned getNumOperands() const
Definition User.h:229
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
Base class of all SIMD vector types.
Type * getElementType() const
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:183
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
IteratorT end() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
Definition ARMBaseInfo.h:49
static ARMCC::CondCodes getSwappedCondition(ARMCC::CondCodes CC)
getSwappedCondition - assume the flags are set by MI(a,b), return the condition code if we modify the...
Definition ARMBaseInfo.h:72
@ SECREL
Thread Pointer Offset.
@ GOT_PREL
Thread Local Storage (General Dynamic Mode)
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo, const LibcallLoweringInfo *libcallLowering)
const unsigned FPReservedBits
const unsigned RoundingBitsPos
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Entry
Definition COFF.h:862
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ Swift
Calling convention for Swift.
Definition CallingConv.h:69
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
Definition CallingConv.h:82
@ PreserveMost
Used for runtime calls that preserves most registers.
Definition CallingConv.h:63
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
Definition CallingConv.h:72
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
Definition CallingConv.h:50
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
Definition CallingConv.h:66
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition CallingConv.h:41
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition CallingConv.h:76
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
Definition CallingConv.h:87
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:516
@ POISON
POISON - A poison node.
Definition ISDOpcodes.h:238
@ SET_FPENV
Sets the current floating-point environment.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition ISDOpcodes.h:170
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition ISDOpcodes.h:543
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition ISDOpcodes.h:174
@ GlobalAddress
Definition ISDOpcodes.h:90
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition ISDOpcodes.h:282
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition ISDOpcodes.h:441
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ GlobalTLSAddress
Definition ISDOpcodes.h:91
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition ISDOpcodes.h:731
@ STRICT_UINT_TO_FP
Definition ISDOpcodes.h:490
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition ISDOpcodes.h:859
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition ISDOpcodes.h:640
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
Definition ISDOpcodes.h:235
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:232
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ CTLS
Count leading redundant sign bits.
Definition ISDOpcodes.h:824
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ STRICT_FP_TO_FP16
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ STRICT_FP16_TO_FP
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition ISDOpcodes.h:141
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition ISDOpcodes.h:226
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:331
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition ISDOpcodes.h:489
@ STRICT_FROUNDEVEN
Definition ISDOpcodes.h:469
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition ISDOpcodes.h:112
@ STRICT_FP_TO_UINT
Definition ISDOpcodes.h:483
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition ISDOpcodes.h:505
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:482
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition ISDOpcodes.h:510
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition ISDOpcodes.h:759
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition ISDOpcodes.h:726
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition ISDOpcodes.h:430
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition ISDOpcodes.h:124
@ STRICT_FNEARBYINT
Definition ISDOpcodes.h:461
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition ISDOpcodes.h:164
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition ISDOpcodes.h:865
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition ISDOpcodes.h:629
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition ISDOpcodes.h:746
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:341
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
constexpr MCPhysReg SP
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
bool sd_match(SDValue N, Pattern &&P)
ConstantInt_match m_ConstInt()
Match any integer constants or splat of an integer constant.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition LLVMContext.h:63
initializer< Ty > init(const Ty &Val)
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
void stable_sort(R &&Range)
Definition STLExtras.h:2132
bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1781
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
InstructionCost Cost
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
RelativeUniformCounterPtr Values
Definition InstrProf.h:91
@ Known
Known to have no common set bits.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Dead
Unused definition.
@ Kill
The last use of a register.
@ Define
Register definition.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:315
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition MathExtras.h:256
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition bit.h:325
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:244
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
void shuffle(Iterator first, Iterator last, RNG &&g)
Definition STLExtras.h:1546
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
Definition MathExtras.h:268
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
bool isReleaseOrStronger(AtomicOrdering AO)
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:149
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:263
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
const unsigned PerfectShuffleTable[6561+1]
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Other
Any other memory.
Definition ModRef.h:68
CombineLevel
Definition DAGCombine.h:15
@ BeforeLegalizeTypes
Definition DAGCombine.h:16
@ AfterLegalizeTypes
Definition DAGCombine.h:17
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
@ FAdd
Sum of floats.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
@ Fast
Assign the register banks as fast as possible (default).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
DWARFExpression::Operation Op
bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr U AbsoluteValue(T X)
Return the absolute value of a signed integer, converted to the corresponding unsigned integer type.
Definition MathExtras.h:592
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
ExceptionHandling
Definition CodeGen.h:54
@ SjLj
setjmp/longjmp based exceptions
Definition CodeGen.h:58
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:2035
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
Definition UndefPoison.h:20
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Definition MathExtras.h:249
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
unsigned gettBLXrOpcode(const MachineFunction &MF)
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
@ Increment
Incrementally increasing token ID.
Definition AllocToken.h:26
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
LLVM_ABI llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
Definition MathExtras.h:199
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition Metadata.h:775
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
static constexpr Align fromLog2(unsigned Log2Value)
Returns an alignment of 1 << Log2Value bytes.
Definition Alignment.h:73
static constexpr DenormalMode getIEEE()
Extended Value Type.
Definition ValueTypes.h:35
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition ValueTypes.h:90
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:509
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition ValueTypes.h:516
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:98
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
bool is128BitVector() const
Return true if this is a 128-bit vector type.
Definition ValueTypes.h:245
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
bool isFixedLengthVector() const
Definition ValueTypes.h:214
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Definition ValueTypes.h:55
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition ValueTypes.h:346
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:499
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Definition ValueTypes.h:240
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
EVT ArgVT
Usually the non-legalized type of the argument, which is the EVT corresponding to the OrigTy IR type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition KnownBits.h:315
unsigned getBitWidth() const
Get the bit width of this value.
Definition KnownBits.h:44
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
Definition KnownBits.h:176
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
Definition KnownBits.h:361
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
Definition KnownBits.h:325
static LLVM_ABI KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
APInt getSignedMinValue() const
Return the minimal signed value possible given these KnownBits.
Definition KnownBits.h:136
Matching combinators.
SmallVector< ArgRegPair, 1 > ArgRegPairs
Vector of call argument and its forwarding register.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
A simple container for information about the supported runtime calls.
LLVM_ABI FunctionCallee getOrInsertLibcallDecl(Module &M, RTLIB::LibcallImpl LibcallImpl) const
LLVM_ABI GlobalValue * getOrInsertImplGlobal(Module &M, RTLIB::LibcallImpl LibcallImpl) const
LLVM_ABI std::pair< FunctionType *, AttributeList > getFunctionTy(LLVMContext &Ctx, const Triple &TT, const DataLayout &DL, RTLIB::LibcallImpl LibcallImpl) const
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
LLVM_ABI void AddToWorklist(SDNode *N)
LLVM_ABI bool recursivelyDeleteUnusedNodes(SDNode *N)
LLVM_ABI SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...