Bug Summary

File:build/source/llvm/lib/IR/AutoUpgrade.cpp
Warning:line 1708, column 32
Division by zero

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name AutoUpgrade.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/IR -I /build/source/llvm/lib/IR -I include -I /build/source/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1683717183 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-05-10-133810-16478-1 -x c++ /build/source/llvm/lib/IR/AutoUpgrade.cpp
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/ADT/StringSwitch.h"
18#include "llvm/BinaryFormat/Dwarf.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/DebugInfo.h"
21#include "llvm/IR/DebugInfoMetadata.h"
22#include "llvm/IR/DiagnosticInfo.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/InstVisitor.h"
26#include "llvm/IR/Instruction.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/IntrinsicsAArch64.h"
30#include "llvm/IR/IntrinsicsARM.h"
31#include "llvm/IR/IntrinsicsWebAssembly.h"
32#include "llvm/IR/IntrinsicsX86.h"
33#include "llvm/IR/LLVMContext.h"
34#include "llvm/IR/Metadata.h"
35#include "llvm/IR/Module.h"
36#include "llvm/IR/Verifier.h"
37#include "llvm/Support/CommandLine.h"
38#include "llvm/Support/ErrorHandling.h"
39#include "llvm/Support/Regex.h"
40#include "llvm/TargetParser/Triple.h"
41#include <cstring>
42
43using namespace llvm;
44
45static cl::opt<bool>
46 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
47 cl::desc("Disable autoupgrade of debug info"));
48
49static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
50
51// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
52// changed their type from v4f32 to v2i64.
53static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID,
54 Function *&NewFn) {
55 // Check whether this is an old version of the function, which received
56 // v4f32 arguments.
57 Type *Arg0Type = F->getFunctionType()->getParamType(0);
58 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
59 return false;
60
61 // Yes, it's old, replace it with new version.
62 rename(F);
63 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
64 return true;
65}
66
67// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
68// arguments have changed their type from i32 to i8.
69static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
70 Function *&NewFn) {
71 // Check that the last argument is an i32.
72 Type *LastArgType = F->getFunctionType()->getParamType(
73 F->getFunctionType()->getNumParams() - 1);
74 if (!LastArgType->isIntegerTy(32))
75 return false;
76
77 // Move this function aside and map down.
78 rename(F);
79 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
80 return true;
81}
82
83// Upgrade the declaration of fp compare intrinsics that change return type
84// from scalar to vXi1 mask.
85static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
86 Function *&NewFn) {
87 // Check if the return type is a vector.
88 if (F->getReturnType()->isVectorTy())
89 return false;
90
91 rename(F);
92 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
93 return true;
94}
95
96static bool UpgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
97 Function *&NewFn) {
98 if (F->getReturnType()->getScalarType()->isBFloatTy())
99 return false;
100
101 rename(F);
102 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
103 return true;
104}
105
106static bool UpgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
107 Function *&NewFn) {
108 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
109 return false;
110
111 rename(F);
112 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
113 return true;
114}
115
116static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
117 // All of the intrinsics matches below should be marked with which llvm
118 // version started autoupgrading them. At some point in the future we would
119 // like to use this information to remove upgrade code for some older
120 // intrinsics. It is currently undecided how we will determine that future
121 // point.
122 if (Name == "addcarryx.u32" || // Added in 8.0
123 Name == "addcarryx.u64" || // Added in 8.0
124 Name == "addcarry.u32" || // Added in 8.0
125 Name == "addcarry.u64" || // Added in 8.0
126 Name == "subborrow.u32" || // Added in 8.0
127 Name == "subborrow.u64" || // Added in 8.0
128 Name.startswith("sse2.padds.") || // Added in 8.0
129 Name.startswith("sse2.psubs.") || // Added in 8.0
130 Name.startswith("sse2.paddus.") || // Added in 8.0
131 Name.startswith("sse2.psubus.") || // Added in 8.0
132 Name.startswith("avx2.padds.") || // Added in 8.0
133 Name.startswith("avx2.psubs.") || // Added in 8.0
134 Name.startswith("avx2.paddus.") || // Added in 8.0
135 Name.startswith("avx2.psubus.") || // Added in 8.0
136 Name.startswith("avx512.padds.") || // Added in 8.0
137 Name.startswith("avx512.psubs.") || // Added in 8.0
138 Name.startswith("avx512.mask.padds.") || // Added in 8.0
139 Name.startswith("avx512.mask.psubs.") || // Added in 8.0
140 Name.startswith("avx512.mask.paddus.") || // Added in 8.0
141 Name.startswith("avx512.mask.psubus.") || // Added in 8.0
142 Name=="ssse3.pabs.b.128" || // Added in 6.0
143 Name=="ssse3.pabs.w.128" || // Added in 6.0
144 Name=="ssse3.pabs.d.128" || // Added in 6.0
145 Name.startswith("fma4.vfmadd.s") || // Added in 7.0
146 Name.startswith("fma.vfmadd.") || // Added in 7.0
147 Name.startswith("fma.vfmsub.") || // Added in 7.0
148 Name.startswith("fma.vfmsubadd.") || // Added in 7.0
149 Name.startswith("fma.vfnmadd.") || // Added in 7.0
150 Name.startswith("fma.vfnmsub.") || // Added in 7.0
151 Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0
152 Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0
153 Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0
154 Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0
155 Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0
156 Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0
157 Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0
158 Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0
159 Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0
160 Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0
161 Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0
162 Name.startswith("avx512.mask.shuf.i") || // Added in 6.0
163 Name.startswith("avx512.mask.shuf.f") || // Added in 6.0
164 Name.startswith("avx512.kunpck") || //added in 6.0
165 Name.startswith("avx2.pabs.") || // Added in 6.0
166 Name.startswith("avx512.mask.pabs.") || // Added in 6.0
167 Name.startswith("avx512.broadcastm") || // Added in 6.0
168 Name == "sse.sqrt.ss" || // Added in 7.0
169 Name == "sse2.sqrt.sd" || // Added in 7.0
170 Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0
171 Name.startswith("avx.sqrt.p") || // Added in 7.0
172 Name.startswith("sse2.sqrt.p") || // Added in 7.0
173 Name.startswith("sse.sqrt.p") || // Added in 7.0
174 Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0
175 Name.startswith("sse2.pcmpeq.") || // Added in 3.1
176 Name.startswith("sse2.pcmpgt.") || // Added in 3.1
177 Name.startswith("avx2.pcmpeq.") || // Added in 3.1
178 Name.startswith("avx2.pcmpgt.") || // Added in 3.1
179 Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9
180 Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9
181 Name.startswith("avx.vperm2f128.") || // Added in 6.0
182 Name == "avx2.vperm2i128" || // Added in 6.0
183 Name == "sse.add.ss" || // Added in 4.0
184 Name == "sse2.add.sd" || // Added in 4.0
185 Name == "sse.sub.ss" || // Added in 4.0
186 Name == "sse2.sub.sd" || // Added in 4.0
187 Name == "sse.mul.ss" || // Added in 4.0
188 Name == "sse2.mul.sd" || // Added in 4.0
189 Name == "sse.div.ss" || // Added in 4.0
190 Name == "sse2.div.sd" || // Added in 4.0
191 Name == "sse41.pmaxsb" || // Added in 3.9
192 Name == "sse2.pmaxs.w" || // Added in 3.9
193 Name == "sse41.pmaxsd" || // Added in 3.9
194 Name == "sse2.pmaxu.b" || // Added in 3.9
195 Name == "sse41.pmaxuw" || // Added in 3.9
196 Name == "sse41.pmaxud" || // Added in 3.9
197 Name == "sse41.pminsb" || // Added in 3.9
198 Name == "sse2.pmins.w" || // Added in 3.9
199 Name == "sse41.pminsd" || // Added in 3.9
200 Name == "sse2.pminu.b" || // Added in 3.9
201 Name == "sse41.pminuw" || // Added in 3.9
202 Name == "sse41.pminud" || // Added in 3.9
203 Name == "avx512.kand.w" || // Added in 7.0
204 Name == "avx512.kandn.w" || // Added in 7.0
205 Name == "avx512.knot.w" || // Added in 7.0
206 Name == "avx512.kor.w" || // Added in 7.0
207 Name == "avx512.kxor.w" || // Added in 7.0
208 Name == "avx512.kxnor.w" || // Added in 7.0
209 Name == "avx512.kortestc.w" || // Added in 7.0
210 Name == "avx512.kortestz.w" || // Added in 7.0
211 Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0
212 Name.startswith("avx2.pmax") || // Added in 3.9
213 Name.startswith("avx2.pmin") || // Added in 3.9
214 Name.startswith("avx512.mask.pmax") || // Added in 4.0
215 Name.startswith("avx512.mask.pmin") || // Added in 4.0
216 Name.startswith("avx2.vbroadcast") || // Added in 3.8
217 Name.startswith("avx2.pbroadcast") || // Added in 3.8
218 Name.startswith("avx.vpermil.") || // Added in 3.1
219 Name.startswith("sse2.pshuf") || // Added in 3.9
220 Name.startswith("avx512.pbroadcast") || // Added in 3.9
221 Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9
222 Name.startswith("avx512.mask.movddup") || // Added in 3.9
223 Name.startswith("avx512.mask.movshdup") || // Added in 3.9
224 Name.startswith("avx512.mask.movsldup") || // Added in 3.9
225 Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9
226 Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9
227 Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9
228 Name.startswith("avx512.mask.shuf.p") || // Added in 4.0
229 Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9
230 Name.startswith("avx512.mask.perm.df.") || // Added in 3.9
231 Name.startswith("avx512.mask.perm.di.") || // Added in 3.9
232 Name.startswith("avx512.mask.punpckl") || // Added in 3.9
233 Name.startswith("avx512.mask.punpckh") || // Added in 3.9
234 Name.startswith("avx512.mask.unpckl.") || // Added in 3.9
235 Name.startswith("avx512.mask.unpckh.") || // Added in 3.9
236 Name.startswith("avx512.mask.pand.") || // Added in 3.9
237 Name.startswith("avx512.mask.pandn.") || // Added in 3.9
238 Name.startswith("avx512.mask.por.") || // Added in 3.9
239 Name.startswith("avx512.mask.pxor.") || // Added in 3.9
240 Name.startswith("avx512.mask.and.") || // Added in 3.9
241 Name.startswith("avx512.mask.andn.") || // Added in 3.9
242 Name.startswith("avx512.mask.or.") || // Added in 3.9
243 Name.startswith("avx512.mask.xor.") || // Added in 3.9
244 Name.startswith("avx512.mask.padd.") || // Added in 4.0
245 Name.startswith("avx512.mask.psub.") || // Added in 4.0
246 Name.startswith("avx512.mask.pmull.") || // Added in 4.0
247 Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
248 Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
249 Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0
250 Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0
251 Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0
252 Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0
253 Name == "avx512.mask.vcvtph2ps.128" || // Added in 11.0
254 Name == "avx512.mask.vcvtph2ps.256" || // Added in 11.0
255 Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0
256 Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0
257 Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0
258 Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0
259 Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0
260 Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0
261 Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0
262 Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0
263 Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0
264 Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0
265 Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0
266 Name == "avx512.cvtusi2sd" || // Added in 7.0
267 Name.startswith("avx512.mask.permvar.") || // Added in 7.0
268 Name == "sse2.pmulu.dq" || // Added in 7.0
269 Name == "sse41.pmuldq" || // Added in 7.0
270 Name == "avx2.pmulu.dq" || // Added in 7.0
271 Name == "avx2.pmul.dq" || // Added in 7.0
272 Name == "avx512.pmulu.dq.512" || // Added in 7.0
273 Name == "avx512.pmul.dq.512" || // Added in 7.0
274 Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0
275 Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0
276 Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0
277 Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0
278 Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0
279 Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0
280 Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0
281 Name.startswith("avx512.mask.packsswb.") || // Added in 5.0
282 Name.startswith("avx512.mask.packssdw.") || // Added in 5.0
283 Name.startswith("avx512.mask.packuswb.") || // Added in 5.0
284 Name.startswith("avx512.mask.packusdw.") || // Added in 5.0
285 Name.startswith("avx512.mask.cmp.b") || // Added in 5.0
286 Name.startswith("avx512.mask.cmp.d") || // Added in 5.0
287 Name.startswith("avx512.mask.cmp.q") || // Added in 5.0
288 Name.startswith("avx512.mask.cmp.w") || // Added in 5.0
289 Name.startswith("avx512.cmp.p") || // Added in 12.0
290 Name.startswith("avx512.mask.ucmp.") || // Added in 5.0
291 Name.startswith("avx512.cvtb2mask.") || // Added in 7.0
292 Name.startswith("avx512.cvtw2mask.") || // Added in 7.0
293 Name.startswith("avx512.cvtd2mask.") || // Added in 7.0
294 Name.startswith("avx512.cvtq2mask.") || // Added in 7.0
295 Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0
296 Name.startswith("avx512.mask.psll.d") || // Added in 4.0
297 Name.startswith("avx512.mask.psll.q") || // Added in 4.0
298 Name.startswith("avx512.mask.psll.w") || // Added in 4.0
299 Name.startswith("avx512.mask.psra.d") || // Added in 4.0
300 Name.startswith("avx512.mask.psra.q") || // Added in 4.0
301 Name.startswith("avx512.mask.psra.w") || // Added in 4.0
302 Name.startswith("avx512.mask.psrl.d") || // Added in 4.0
303 Name.startswith("avx512.mask.psrl.q") || // Added in 4.0
304 Name.startswith("avx512.mask.psrl.w") || // Added in 4.0
305 Name.startswith("avx512.mask.pslli") || // Added in 4.0
306 Name.startswith("avx512.mask.psrai") || // Added in 4.0
307 Name.startswith("avx512.mask.psrli") || // Added in 4.0
308 Name.startswith("avx512.mask.psllv") || // Added in 4.0
309 Name.startswith("avx512.mask.psrav") || // Added in 4.0
310 Name.startswith("avx512.mask.psrlv") || // Added in 4.0
311 Name.startswith("sse41.pmovsx") || // Added in 3.8
312 Name.startswith("sse41.pmovzx") || // Added in 3.9
313 Name.startswith("avx2.pmovsx") || // Added in 3.9
314 Name.startswith("avx2.pmovzx") || // Added in 3.9
315 Name.startswith("avx512.mask.pmovsx") || // Added in 4.0
316 Name.startswith("avx512.mask.pmovzx") || // Added in 4.0
317 Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0
318 Name.startswith("avx512.mask.pternlog.") || // Added in 7.0
319 Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0
320 Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0
321 Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0
322 Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0
323 Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0
324 Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0
325 Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0
326 Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0
327 Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0
328 Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0
329 Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0
330 Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0
331 Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0
332 Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0
333 Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0
334 Name.startswith("avx512.mask.vpshld.") || // Added in 7.0
335 Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0
336 Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0
337 Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0
338 Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0
339 Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0
340 Name.startswith("avx512.vpshld.") || // Added in 8.0
341 Name.startswith("avx512.vpshrd.") || // Added in 8.0
342 Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0
343 Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0
344 Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0
345 Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0
346 Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0
347 Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0
348 Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0
349 Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0
350 Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0
351 Name.startswith("avx512.mask.conflict.") || // Added in 9.0
352 Name == "avx512.mask.pmov.qd.256" || // Added in 9.0
353 Name == "avx512.mask.pmov.qd.512" || // Added in 9.0
354 Name == "avx512.mask.pmov.wb.256" || // Added in 9.0
355 Name == "avx512.mask.pmov.wb.512" || // Added in 9.0
356 Name == "sse.cvtsi2ss" || // Added in 7.0
357 Name == "sse.cvtsi642ss" || // Added in 7.0
358 Name == "sse2.cvtsi2sd" || // Added in 7.0
359 Name == "sse2.cvtsi642sd" || // Added in 7.0
360 Name == "sse2.cvtss2sd" || // Added in 7.0
361 Name == "sse2.cvtdq2pd" || // Added in 3.9
362 Name == "sse2.cvtdq2ps" || // Added in 7.0
363 Name == "sse2.cvtps2pd" || // Added in 3.9
364 Name == "avx.cvtdq2.pd.256" || // Added in 3.9
365 Name == "avx.cvtdq2.ps.256" || // Added in 7.0
366 Name == "avx.cvt.ps2.pd.256" || // Added in 3.9
367 Name.startswith("vcvtph2ps.") || // Added in 11.0
368 Name.startswith("avx.vinsertf128.") || // Added in 3.7
369 Name == "avx2.vinserti128" || // Added in 3.7
370 Name.startswith("avx512.mask.insert") || // Added in 4.0
371 Name.startswith("avx.vextractf128.") || // Added in 3.7
372 Name == "avx2.vextracti128" || // Added in 3.7
373 Name.startswith("avx512.mask.vextract") || // Added in 4.0
374 Name.startswith("sse4a.movnt.") || // Added in 3.9
375 Name.startswith("avx.movnt.") || // Added in 3.2
376 Name.startswith("avx512.storent.") || // Added in 3.9
377 Name == "sse41.movntdqa" || // Added in 5.0
378 Name == "avx2.movntdqa" || // Added in 5.0
379 Name == "avx512.movntdqa" || // Added in 5.0
380 Name == "sse2.storel.dq" || // Added in 3.9
381 Name.startswith("sse.storeu.") || // Added in 3.9
382 Name.startswith("sse2.storeu.") || // Added in 3.9
383 Name.startswith("avx.storeu.") || // Added in 3.9
384 Name.startswith("avx512.mask.storeu.") || // Added in 3.9
385 Name.startswith("avx512.mask.store.p") || // Added in 3.9
386 Name.startswith("avx512.mask.store.b.") || // Added in 3.9
387 Name.startswith("avx512.mask.store.w.") || // Added in 3.9
388 Name.startswith("avx512.mask.store.d.") || // Added in 3.9
389 Name.startswith("avx512.mask.store.q.") || // Added in 3.9
390 Name == "avx512.mask.store.ss" || // Added in 7.0
391 Name.startswith("avx512.mask.loadu.") || // Added in 3.9
392 Name.startswith("avx512.mask.load.") || // Added in 3.9
393 Name.startswith("avx512.mask.expand.load.") || // Added in 7.0
394 Name.startswith("avx512.mask.compress.store.") || // Added in 7.0
395 Name.startswith("avx512.mask.expand.b") || // Added in 9.0
396 Name.startswith("avx512.mask.expand.w") || // Added in 9.0
397 Name.startswith("avx512.mask.expand.d") || // Added in 9.0
398 Name.startswith("avx512.mask.expand.q") || // Added in 9.0
399 Name.startswith("avx512.mask.expand.p") || // Added in 9.0
400 Name.startswith("avx512.mask.compress.b") || // Added in 9.0
401 Name.startswith("avx512.mask.compress.w") || // Added in 9.0
402 Name.startswith("avx512.mask.compress.d") || // Added in 9.0
403 Name.startswith("avx512.mask.compress.q") || // Added in 9.0
404 Name.startswith("avx512.mask.compress.p") || // Added in 9.0
405 Name == "sse42.crc32.64.8" || // Added in 3.4
406 Name.startswith("avx.vbroadcast.s") || // Added in 3.5
407 Name.startswith("avx512.vbroadcast.s") || // Added in 7.0
408 Name.startswith("avx512.mask.palignr.") || // Added in 3.9
409 Name.startswith("avx512.mask.valign.") || // Added in 4.0
410 Name.startswith("sse2.psll.dq") || // Added in 3.7
411 Name.startswith("sse2.psrl.dq") || // Added in 3.7
412 Name.startswith("avx2.psll.dq") || // Added in 3.7
413 Name.startswith("avx2.psrl.dq") || // Added in 3.7
414 Name.startswith("avx512.psll.dq") || // Added in 3.9
415 Name.startswith("avx512.psrl.dq") || // Added in 3.9
416 Name == "sse41.pblendw" || // Added in 3.7
417 Name.startswith("sse41.blendp") || // Added in 3.7
418 Name.startswith("avx.blend.p") || // Added in 3.7
419 Name == "avx2.pblendw" || // Added in 3.7
420 Name.startswith("avx2.pblendd.") || // Added in 3.7
421 Name.startswith("avx.vbroadcastf128") || // Added in 4.0
422 Name == "avx2.vbroadcasti128" || // Added in 3.7
423 Name.startswith("avx512.mask.broadcastf32x4.") || // Added in 6.0
424 Name.startswith("avx512.mask.broadcastf64x2.") || // Added in 6.0
425 Name.startswith("avx512.mask.broadcastf32x8.") || // Added in 6.0
426 Name.startswith("avx512.mask.broadcastf64x4.") || // Added in 6.0
427 Name.startswith("avx512.mask.broadcasti32x4.") || // Added in 6.0
428 Name.startswith("avx512.mask.broadcasti64x2.") || // Added in 6.0
429 Name.startswith("avx512.mask.broadcasti32x8.") || // Added in 6.0
430 Name.startswith("avx512.mask.broadcasti64x4.") || // Added in 6.0
431 Name == "xop.vpcmov" || // Added in 3.8
432 Name == "xop.vpcmov.256" || // Added in 5.0
433 Name.startswith("avx512.mask.move.s") || // Added in 4.0
434 Name.startswith("avx512.cvtmask2") || // Added in 5.0
435 Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0
436 Name.startswith("xop.vprot") || // Added in 8.0
437 Name.startswith("avx512.prol") || // Added in 8.0
438 Name.startswith("avx512.pror") || // Added in 8.0
439 Name.startswith("avx512.mask.prorv.") || // Added in 8.0
440 Name.startswith("avx512.mask.pror.") || // Added in 8.0
441 Name.startswith("avx512.mask.prolv.") || // Added in 8.0
442 Name.startswith("avx512.mask.prol.") || // Added in 8.0
443 Name.startswith("avx512.ptestm") || //Added in 6.0
444 Name.startswith("avx512.ptestnm") || //Added in 6.0
445 Name.startswith("avx512.mask.pavg")) // Added in 6.0
446 return true;
447
448 return false;
449}
450
451static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
452 Function *&NewFn) {
453 // Only handle intrinsics that start with "x86.".
454 if (!Name.startswith("x86."))
455 return false;
456 // Remove "x86." prefix.
457 Name = Name.substr(4);
458
459 if (ShouldUpgradeX86Intrinsic(F, Name)) {
460 NewFn = nullptr;
461 return true;
462 }
463
464 if (Name == "rdtscp") { // Added in 8.0
465 // If this intrinsic has 0 operands, it's the new version.
466 if (F->getFunctionType()->getNumParams() == 0)
467 return false;
468
469 rename(F);
470 NewFn = Intrinsic::getDeclaration(F->getParent(),
471 Intrinsic::x86_rdtscp);
472 return true;
473 }
474
475 // SSE4.1 ptest functions may have an old signature.
476 if (Name.startswith("sse41.ptest")) { // Added in 3.2
477 if (Name.substr(11) == "c")
478 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn);
479 if (Name.substr(11) == "z")
480 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn);
481 if (Name.substr(11) == "nzc")
482 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
483 }
484 // Several blend and other instructions with masks used the wrong number of
485 // bits.
486 if (Name == "sse41.insertps") // Added in 3.6
487 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
488 NewFn);
489 if (Name == "sse41.dppd") // Added in 3.6
490 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
491 NewFn);
492 if (Name == "sse41.dpps") // Added in 3.6
493 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
494 NewFn);
495 if (Name == "sse41.mpsadbw") // Added in 3.6
496 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
497 NewFn);
498 if (Name == "avx.dp.ps.256") // Added in 3.6
499 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
500 NewFn);
501 if (Name == "avx2.mpsadbw") // Added in 3.6
502 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
503 NewFn);
504 if (Name == "avx512.mask.cmp.pd.128") // Added in 7.0
505 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128,
506 NewFn);
507 if (Name == "avx512.mask.cmp.pd.256") // Added in 7.0
508 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256,
509 NewFn);
510 if (Name == "avx512.mask.cmp.pd.512") // Added in 7.0
511 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
512 NewFn);
513 if (Name == "avx512.mask.cmp.ps.128") // Added in 7.0
514 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128,
515 NewFn);
516 if (Name == "avx512.mask.cmp.ps.256") // Added in 7.0
517 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256,
518 NewFn);
519 if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0
520 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
521 NewFn);
522 if (Name == "avx512bf16.cvtne2ps2bf16.128") // Added in 9.0
523 return UpgradeX86BF16Intrinsic(
524 F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128, NewFn);
525 if (Name == "avx512bf16.cvtne2ps2bf16.256") // Added in 9.0
526 return UpgradeX86BF16Intrinsic(
527 F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256, NewFn);
528 if (Name == "avx512bf16.cvtne2ps2bf16.512") // Added in 9.0
529 return UpgradeX86BF16Intrinsic(
530 F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512, NewFn);
531 if (Name == "avx512bf16.mask.cvtneps2bf16.128") // Added in 9.0
532 return UpgradeX86BF16Intrinsic(
533 F, Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128, NewFn);
534 if (Name == "avx512bf16.cvtneps2bf16.256") // Added in 9.0
535 return UpgradeX86BF16Intrinsic(
536 F, Intrinsic::x86_avx512bf16_cvtneps2bf16_256, NewFn);
537 if (Name == "avx512bf16.cvtneps2bf16.512") // Added in 9.0
538 return UpgradeX86BF16Intrinsic(
539 F, Intrinsic::x86_avx512bf16_cvtneps2bf16_512, NewFn);
540 if (Name == "avx512bf16.dpbf16ps.128") // Added in 9.0
541 return UpgradeX86BF16DPIntrinsic(
542 F, Intrinsic::x86_avx512bf16_dpbf16ps_128, NewFn);
543 if (Name == "avx512bf16.dpbf16ps.256") // Added in 9.0
544 return UpgradeX86BF16DPIntrinsic(
545 F, Intrinsic::x86_avx512bf16_dpbf16ps_256, NewFn);
546 if (Name == "avx512bf16.dpbf16ps.512") // Added in 9.0
547 return UpgradeX86BF16DPIntrinsic(
548 F, Intrinsic::x86_avx512bf16_dpbf16ps_512, NewFn);
549
550 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
551 if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
552 rename(F);
553 NewFn = Intrinsic::getDeclaration(F->getParent(),
554 Intrinsic::x86_xop_vfrcz_ss);
555 return true;
556 }
557 if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {
558 rename(F);
559 NewFn = Intrinsic::getDeclaration(F->getParent(),
560 Intrinsic::x86_xop_vfrcz_sd);
561 return true;
562 }
563 // Upgrade any XOP PERMIL2 index operand still using a float/double vector.
564 if (Name.startswith("xop.vpermil2")) { // Added in 3.9
565 auto Idx = F->getFunctionType()->getParamType(2);
566 if (Idx->isFPOrFPVectorTy()) {
567 rename(F);
568 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
569 unsigned EltSize = Idx->getScalarSizeInBits();
570 Intrinsic::ID Permil2ID;
571 if (EltSize == 64 && IdxSize == 128)
572 Permil2ID = Intrinsic::x86_xop_vpermil2pd;
573 else if (EltSize == 32 && IdxSize == 128)
574 Permil2ID = Intrinsic::x86_xop_vpermil2ps;
575 else if (EltSize == 64 && IdxSize == 256)
576 Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
577 else
578 Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
579 NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);
580 return true;
581 }
582 }
583
584 if (Name == "seh.recoverfp") {
585 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
586 return true;
587 }
588
589 return false;
590}
591
592static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
593 assert(F && "Illegal to upgrade a non-existent Function.")(static_cast <bool> (F && "Illegal to upgrade a non-existent Function."
) ? void (0) : __assert_fail ("F && \"Illegal to upgrade a non-existent Function.\""
, "llvm/lib/IR/AutoUpgrade.cpp", 593, __extension__ __PRETTY_FUNCTION__
));
594
595 // Quickly eliminate it, if it's not a candidate.
596 StringRef Name = F->getName();
597 if (Name.size() <= 7 || !Name.startswith("llvm."))
598 return false;
599 Name = Name.substr(5); // Strip off "llvm."
600
601 switch (Name[0]) {
602 default: break;
603 case 'a': {
604 if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) {
605 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
606 F->arg_begin()->getType());
607 return true;
608 }
609 if (Name.startswith("aarch64.neon.frintn")) {
610 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::roundeven,
611 F->arg_begin()->getType());
612 return true;
613 }
614 if (Name.startswith("aarch64.neon.rbit")) {
615 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
616 F->arg_begin()->getType());
617 return true;
618 }
619 if (Name == "aarch64.sve.bfdot.lane") {
620 NewFn = Intrinsic::getDeclaration(F->getParent(),
621 Intrinsic::aarch64_sve_bfdot_lane_v2);
622 return true;
623 }
624 if (Name == "aarch64.sve.bfmlalb.lane") {
625 NewFn = Intrinsic::getDeclaration(F->getParent(),
626 Intrinsic::aarch64_sve_bfmlalb_lane_v2);
627 return true;
628 }
629 if (Name == "aarch64.sve.bfmlalt.lane") {
630 NewFn = Intrinsic::getDeclaration(F->getParent(),
631 Intrinsic::aarch64_sve_bfmlalt_lane_v2);
632 return true;
633 }
634 static const Regex LdRegex("^aarch64\\.sve\\.ld[234](.nxv[a-z0-9]+|$)");
635 if (LdRegex.match(Name)) {
636 Type *ScalarTy =
637 dyn_cast<VectorType>(F->getReturnType())->getElementType();
638 ElementCount EC =
639 dyn_cast<VectorType>(F->arg_begin()->getType())->getElementCount();
640 Type *Ty = VectorType::get(ScalarTy, EC);
641 Intrinsic::ID ID =
642 StringSwitch<Intrinsic::ID>(Name)
643 .StartsWith("aarch64.sve.ld2", Intrinsic::aarch64_sve_ld2_sret)
644 .StartsWith("aarch64.sve.ld3", Intrinsic::aarch64_sve_ld3_sret)
645 .StartsWith("aarch64.sve.ld4", Intrinsic::aarch64_sve_ld4_sret)
646 .Default(Intrinsic::not_intrinsic);
647 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Ty);
648 return true;
649 }
650 if (Name.startswith("aarch64.sve.tuple.get")) {
651 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
652 NewFn = Intrinsic::getDeclaration(F->getParent(),
653 Intrinsic::vector_extract, Tys);
654 return true;
655 }
656 if (Name.startswith("aarch64.sve.tuple.set")) {
657 auto Args = F->getFunctionType()->params();
658 Type *Tys[] = {Args[0], Args[2], Args[1]};
659 NewFn = Intrinsic::getDeclaration(F->getParent(),
660 Intrinsic::vector_insert, Tys);
661 return true;
662 }
663 static const Regex CreateTupleRegex(
664 "^aarch64\\.sve\\.tuple\\.create[234](.nxv[a-z0-9]+|$)");
665 if (CreateTupleRegex.match(Name)) {
666 auto Args = F->getFunctionType()->params();
667 Type *Tys[] = {F->getReturnType(), Args[1]};
668 NewFn = Intrinsic::getDeclaration(F->getParent(),
669 Intrinsic::vector_insert, Tys);
670 return true;
671 }
672 if (Name.startswith("arm.neon.vclz")) {
673 Type* args[2] = {
674 F->arg_begin()->getType(),
675 Type::getInt1Ty(F->getContext())
676 };
677 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
678 // the end of the name. Change name from llvm.arm.neon.vclz.* to
679 // llvm.ctlz.*
680 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
681 NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
682 "llvm.ctlz." + Name.substr(14), F->getParent());
683 return true;
684 }
685 if (Name.startswith("arm.neon.vcnt")) {
686 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
687 F->arg_begin()->getType());
688 return true;
689 }
690 static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
691 if (vstRegex.match(Name)) {
692 static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
693 Intrinsic::arm_neon_vst2,
694 Intrinsic::arm_neon_vst3,
695 Intrinsic::arm_neon_vst4};
696
697 static const Intrinsic::ID StoreLaneInts[] = {
698 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
699 Intrinsic::arm_neon_vst4lane
700 };
701
702 auto fArgs = F->getFunctionType()->params();
703 Type *Tys[] = {fArgs[0], fArgs[1]};
704 if (!Name.contains("lane"))
705 NewFn = Intrinsic::getDeclaration(F->getParent(),
706 StoreInts[fArgs.size() - 3], Tys);
707 else
708 NewFn = Intrinsic::getDeclaration(F->getParent(),
709 StoreLaneInts[fArgs.size() - 5], Tys);
710 return true;
711 }
712 if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
713 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
714 return true;
715 }
716 if (Name.startswith("arm.neon.vqadds.")) {
717 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,
718 F->arg_begin()->getType());
719 return true;
720 }
721 if (Name.startswith("arm.neon.vqaddu.")) {
722 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,
723 F->arg_begin()->getType());
724 return true;
725 }
726 if (Name.startswith("arm.neon.vqsubs.")) {
727 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,
728 F->arg_begin()->getType());
729 return true;
730 }
731 if (Name.startswith("arm.neon.vqsubu.")) {
732 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,
733 F->arg_begin()->getType());
734 return true;
735 }
736 if (Name.startswith("aarch64.neon.addp")) {
737 if (F->arg_size() != 2)
738 break; // Invalid IR.
739 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
740 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
741 NewFn = Intrinsic::getDeclaration(F->getParent(),
742 Intrinsic::aarch64_neon_faddp, Ty);
743 return true;
744 }
745 }
746
747 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8
748 // respectively
749 if ((Name.startswith("arm.neon.bfdot.") ||
750 Name.startswith("aarch64.neon.bfdot.")) &&
751 Name.endswith("i8")) {
752 Intrinsic::ID IID =
753 StringSwitch<Intrinsic::ID>(Name)
754 .Cases("arm.neon.bfdot.v2f32.v8i8",
755 "arm.neon.bfdot.v4f32.v16i8",
756 Intrinsic::arm_neon_bfdot)
757 .Cases("aarch64.neon.bfdot.v2f32.v8i8",
758 "aarch64.neon.bfdot.v4f32.v16i8",
759 Intrinsic::aarch64_neon_bfdot)
760 .Default(Intrinsic::not_intrinsic);
761 if (IID == Intrinsic::not_intrinsic)
762 break;
763
764 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
765 assert((OperandWidth == 64 || OperandWidth == 128) &&(static_cast <bool> ((OperandWidth == 64 || OperandWidth
== 128) && "Unexpected operand width") ? void (0) : __assert_fail
("(OperandWidth == 64 || OperandWidth == 128) && \"Unexpected operand width\""
, "llvm/lib/IR/AutoUpgrade.cpp", 766, __extension__ __PRETTY_FUNCTION__
))
766 "Unexpected operand width")(static_cast <bool> ((OperandWidth == 64 || OperandWidth
== 128) && "Unexpected operand width") ? void (0) : __assert_fail
("(OperandWidth == 64 || OperandWidth == 128) && \"Unexpected operand width\""
, "llvm/lib/IR/AutoUpgrade.cpp", 766, __extension__ __PRETTY_FUNCTION__
));
767 LLVMContext &Ctx = F->getParent()->getContext();
768 std::array<Type *, 2> Tys {{
769 F->getReturnType(),
770 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)
771 }};
772 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
773 return true;
774 }
775
776 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore
777 // and accept v8bf16 instead of v16i8
778 if ((Name.startswith("arm.neon.bfm") ||
779 Name.startswith("aarch64.neon.bfm")) &&
780 Name.endswith(".v4f32.v16i8")) {
781 Intrinsic::ID IID =
782 StringSwitch<Intrinsic::ID>(Name)
783 .Case("arm.neon.bfmmla.v4f32.v16i8",
784 Intrinsic::arm_neon_bfmmla)
785 .Case("arm.neon.bfmlalb.v4f32.v16i8",
786 Intrinsic::arm_neon_bfmlalb)
787 .Case("arm.neon.bfmlalt.v4f32.v16i8",
788 Intrinsic::arm_neon_bfmlalt)
789 .Case("aarch64.neon.bfmmla.v4f32.v16i8",
790 Intrinsic::aarch64_neon_bfmmla)
791 .Case("aarch64.neon.bfmlalb.v4f32.v16i8",
792 Intrinsic::aarch64_neon_bfmlalb)
793 .Case("aarch64.neon.bfmlalt.v4f32.v16i8",
794 Intrinsic::aarch64_neon_bfmlalt)
795 .Default(Intrinsic::not_intrinsic);
796 if (IID == Intrinsic::not_intrinsic)
797 break;
798
799 std::array<Type *, 0> Tys;
800 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
801 return true;
802 }
803
804 if (Name == "arm.mve.vctp64" &&
805 cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
806 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename the
807 // function and deal with it below in UpgradeIntrinsicCall.
808 rename(F);
809 return true;
810 }
811 // These too are changed to accept a v2i1 insteead of the old v4i1.
812 if (Name == "arm.mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
813 Name == "arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
814 Name == "arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
815 Name == "arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
816 Name ==
817 "arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
818 Name == "arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
819 Name == "arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
820 Name == "arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
821 Name ==
822 "arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
823 Name == "arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
824 Name == "arm.cde.vcx1q.predicated.v2i64.v4i1" ||
825 Name == "arm.cde.vcx1qa.predicated.v2i64.v4i1" ||
826 Name == "arm.cde.vcx2q.predicated.v2i64.v4i1" ||
827 Name == "arm.cde.vcx2qa.predicated.v2i64.v4i1" ||
828 Name == "arm.cde.vcx3q.predicated.v2i64.v4i1" ||
829 Name == "arm.cde.vcx3qa.predicated.v2i64.v4i1")
830 return true;
831
832 if (Name == "amdgcn.alignbit") {
833 // Target specific intrinsic became redundant
834 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::fshr,
835 {F->getReturnType()});
836 return true;
837 }
838
839 break;
840 }
841
842 case 'c': {
843 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
844 rename(F);
845 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
846 F->arg_begin()->getType());
847 return true;
848 }
849 if (Name.startswith("cttz.") && F->arg_size() == 1) {
850 rename(F);
851 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
852 F->arg_begin()->getType());
853 return true;
854 }
855 break;
856 }
857 case 'd': {
858 if (Name == "dbg.addr") {
859 rename(F);
860 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
861 return true;
862 }
863 if (Name == "dbg.value" && F->arg_size() == 4) {
864 rename(F);
865 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
866 return true;
867 }
868 break;
869 }
870 case 'e': {
871 if (Name.startswith("experimental.vector.extract.")) {
872 rename(F);
873 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
874 NewFn = Intrinsic::getDeclaration(F->getParent(),
875 Intrinsic::vector_extract, Tys);
876 return true;
877 }
878
879 if (Name.startswith("experimental.vector.insert.")) {
880 rename(F);
881 auto Args = F->getFunctionType()->params();
882 Type *Tys[] = {Args[0], Args[1]};
883 NewFn = Intrinsic::getDeclaration(F->getParent(),
884 Intrinsic::vector_insert, Tys);
885 return true;
886 }
887
888 SmallVector<StringRef, 2> Groups;
889 static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[a-z][0-9]+");
890 if (R.match(Name, &Groups)) {
891 Intrinsic::ID ID;
892 ID = StringSwitch<Intrinsic::ID>(Groups[1])
893 .Case("add", Intrinsic::vector_reduce_add)
894 .Case("mul", Intrinsic::vector_reduce_mul)
895 .Case("and", Intrinsic::vector_reduce_and)
896 .Case("or", Intrinsic::vector_reduce_or)
897 .Case("xor", Intrinsic::vector_reduce_xor)
898 .Case("smax", Intrinsic::vector_reduce_smax)
899 .Case("smin", Intrinsic::vector_reduce_smin)
900 .Case("umax", Intrinsic::vector_reduce_umax)
901 .Case("umin", Intrinsic::vector_reduce_umin)
902 .Case("fmax", Intrinsic::vector_reduce_fmax)
903 .Case("fmin", Intrinsic::vector_reduce_fmin)
904 .Default(Intrinsic::not_intrinsic);
905 if (ID != Intrinsic::not_intrinsic) {
906 rename(F);
907 auto Args = F->getFunctionType()->params();
908 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, {Args[0]});
909 return true;
910 }
911 }
912 static const Regex R2(
913 "^experimental.vector.reduce.v2.([a-z]+)\\.[fi][0-9]+");
914 Groups.clear();
915 if (R2.match(Name, &Groups)) {
916 Intrinsic::ID ID = Intrinsic::not_intrinsic;
917 if (Groups[1] == "fadd")
918 ID = Intrinsic::vector_reduce_fadd;
919 if (Groups[1] == "fmul")
920 ID = Intrinsic::vector_reduce_fmul;
921 if (ID != Intrinsic::not_intrinsic) {
922 rename(F);
923 auto Args = F->getFunctionType()->params();
924 Type *Tys[] = {Args[1]};
925 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
926 return true;
927 }
928 }
929 break;
930 }
931 case 'f':
932 if (Name.startswith("flt.rounds")) {
933 rename(F);
934 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::get_rounding);
935 return true;
936 }
937 break;
938 case 'i':
939 case 'l': {
940 bool IsLifetimeStart = Name.startswith("lifetime.start");
941 if (IsLifetimeStart || Name.startswith("invariant.start")) {
942 Intrinsic::ID ID = IsLifetimeStart ?
943 Intrinsic::lifetime_start : Intrinsic::invariant_start;
944 auto Args = F->getFunctionType()->params();
945 Type* ObjectPtr[1] = {Args[1]};
946 if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) {
947 rename(F);
948 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
949 return true;
950 }
951 }
952
953 bool IsLifetimeEnd = Name.startswith("lifetime.end");
954 if (IsLifetimeEnd || Name.startswith("invariant.end")) {
955 Intrinsic::ID ID = IsLifetimeEnd ?
956 Intrinsic::lifetime_end : Intrinsic::invariant_end;
957
958 auto Args = F->getFunctionType()->params();
959 Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]};
960 if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) {
961 rename(F);
962 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
963 return true;
964 }
965 }
966 if (Name.startswith("invariant.group.barrier")) {
967 // Rename invariant.group.barrier to launder.invariant.group
968 auto Args = F->getFunctionType()->params();
969 Type* ObjectPtr[1] = {Args[0]};
970 rename(F);
971 NewFn = Intrinsic::getDeclaration(F->getParent(),
972 Intrinsic::launder_invariant_group, ObjectPtr);
973 return true;
974
975 }
976
977 break;
978 }
979 case 'm': {
980 if (Name.startswith("masked.load.")) {
981 Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };
982 if (F->getName() !=
983 Intrinsic::getName(Intrinsic::masked_load, Tys, F->getParent())) {
984 rename(F);
985 NewFn = Intrinsic::getDeclaration(F->getParent(),
986 Intrinsic::masked_load,
987 Tys);
988 return true;
989 }
990 }
991 if (Name.startswith("masked.store.")) {
992 auto Args = F->getFunctionType()->params();
993 Type *Tys[] = { Args[0], Args[1] };
994 if (F->getName() !=
995 Intrinsic::getName(Intrinsic::masked_store, Tys, F->getParent())) {
996 rename(F);
997 NewFn = Intrinsic::getDeclaration(F->getParent(),
998 Intrinsic::masked_store,
999 Tys);
1000 return true;
1001 }
1002 }
1003 // Renaming gather/scatter intrinsics with no address space overloading
1004 // to the new overload which includes an address space
1005 if (Name.startswith("masked.gather.")) {
1006 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1007 if (F->getName() !=
1008 Intrinsic::getName(Intrinsic::masked_gather, Tys, F->getParent())) {
1009 rename(F);
1010 NewFn = Intrinsic::getDeclaration(F->getParent(),
1011 Intrinsic::masked_gather, Tys);
1012 return true;
1013 }
1014 }
1015 if (Name.startswith("masked.scatter.")) {
1016 auto Args = F->getFunctionType()->params();
1017 Type *Tys[] = {Args[0], Args[1]};
1018 if (F->getName() !=
1019 Intrinsic::getName(Intrinsic::masked_scatter, Tys, F->getParent())) {
1020 rename(F);
1021 NewFn = Intrinsic::getDeclaration(F->getParent(),
1022 Intrinsic::masked_scatter, Tys);
1023 return true;
1024 }
1025 }
1026 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1027 // alignment parameter to embedding the alignment as an attribute of
1028 // the pointer args.
1029 if (Name.startswith("memcpy.") && F->arg_size() == 5) {
1030 rename(F);
1031 // Get the types of dest, src, and len
1032 ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
1033 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy,
1034 ParamTypes);
1035 return true;
1036 }
1037 if (Name.startswith("memmove.") && F->arg_size() == 5) {
1038 rename(F);
1039 // Get the types of dest, src, and len
1040 ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
1041 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove,
1042 ParamTypes);
1043 return true;
1044 }
1045 if (Name.startswith("memset.") && F->arg_size() == 5) {
1046 rename(F);
1047 // Get the types of dest, and len
1048 const auto *FT = F->getFunctionType();
1049 Type *ParamTypes[2] = {
1050 FT->getParamType(0), // Dest
1051 FT->getParamType(2) // len
1052 };
1053 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,
1054 ParamTypes);
1055 return true;
1056 }
1057 break;
1058 }
1059 case 'n': {
1060 if (Name.startswith("nvvm.")) {
1061 Name = Name.substr(5);
1062
1063 // The following nvvm intrinsics correspond exactly to an LLVM intrinsic.
1064 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name)
1065 .Cases("brev32", "brev64", Intrinsic::bitreverse)
1066 .Case("clz.i", Intrinsic::ctlz)
1067 .Case("popc.i", Intrinsic::ctpop)
1068 .Default(Intrinsic::not_intrinsic);
1069 if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) {
1070 NewFn = Intrinsic::getDeclaration(F->getParent(), IID,
1071 {F->getReturnType()});
1072 return true;
1073 }
1074
1075 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1076 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1077 //
1078 // TODO: We could add lohi.i2d.
1079 bool Expand = StringSwitch<bool>(Name)
1080 .Cases("abs.i", "abs.ll", true)
1081 .Cases("clz.ll", "popc.ll", "h2f", true)
1082 .Cases("max.i", "max.ll", "max.ui", "max.ull", true)
1083 .Cases("min.i", "min.ll", "min.ui", "min.ull", true)
1084 .StartsWith("atomic.load.add.f32.p", true)
1085 .StartsWith("atomic.load.add.f64.p", true)
1086 .Default(false);
1087 if (Expand) {
1088 NewFn = nullptr;
1089 return true;
1090 }
1091 }
1092 break;
1093 }
1094 case 'o':
1095 // We only need to change the name to match the mangling including the
1096 // address space.
1097 if (Name.startswith("objectsize.")) {
1098 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
1099 if (F->arg_size() == 2 || F->arg_size() == 3 ||
1100 F->getName() !=
1101 Intrinsic::getName(Intrinsic::objectsize, Tys, F->getParent())) {
1102 rename(F);
1103 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
1104 Tys);
1105 return true;
1106 }
1107 }
1108 break;
1109
1110 case 'p':
1111 if (Name == "prefetch") {
1112 // Handle address space overloading.
1113 Type *Tys[] = {F->arg_begin()->getType()};
1114 if (F->getName() !=
1115 Intrinsic::getName(Intrinsic::prefetch, Tys, F->getParent())) {
1116 rename(F);
1117 NewFn =
1118 Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys);
1119 return true;
1120 }
1121 } else if (Name.startswith("ptr.annotation.") && F->arg_size() == 4) {
1122 rename(F);
1123 NewFn = Intrinsic::getDeclaration(
1124 F->getParent(), Intrinsic::ptr_annotation,
1125 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1126 return true;
1127 }
1128 break;
1129
1130 case 's':
1131 if (Name == "stackprotectorcheck") {
1132 NewFn = nullptr;
1133 return true;
1134 }
1135 break;
1136
1137 case 'v': {
1138 if (Name == "var.annotation" && F->arg_size() == 4) {
1139 rename(F);
1140 NewFn = Intrinsic::getDeclaration(
1141 F->getParent(), Intrinsic::var_annotation,
1142 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1143 return true;
1144 }
1145 break;
1146 }
1147
1148 case 'w':
1149 if (Name.startswith("wasm.fma.")) {
1150 rename(F);
1151 NewFn = Intrinsic::getDeclaration(
1152 F->getParent(), Intrinsic::wasm_relaxed_madd, F->getReturnType());
1153 return true;
1154 }
1155 if (Name.startswith("wasm.fms.")) {
1156 rename(F);
1157 NewFn = Intrinsic::getDeclaration(
1158 F->getParent(), Intrinsic::wasm_relaxed_nmadd, F->getReturnType());
1159 return true;
1160 }
1161 if (Name.startswith("wasm.laneselect.")) {
1162 rename(F);
1163 NewFn = Intrinsic::getDeclaration(
1164 F->getParent(), Intrinsic::wasm_relaxed_laneselect,
1165 F->getReturnType());
1166 return true;
1167 }
1168 if (Name == "wasm.dot.i8x16.i7x16.signed") {
1169 rename(F);
1170 NewFn = Intrinsic::getDeclaration(
1171 F->getParent(), Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed);
1172 return true;
1173 }
1174 if (Name == "wasm.dot.i8x16.i7x16.add.signed") {
1175 rename(F);
1176 NewFn = Intrinsic::getDeclaration(
1177 F->getParent(), Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
1178 return true;
1179 }
1180 break;
1181
1182 case 'x':
1183 if (UpgradeX86IntrinsicFunction(F, Name, NewFn))
1184 return true;
1185 }
1186
1187 auto *ST = dyn_cast<StructType>(F->getReturnType());
1188 if (ST && (!ST->isLiteral() || ST->isPacked())) {
1189 // Replace return type with literal non-packed struct. Only do this for
1190 // intrinsics declared to return a struct, not for intrinsics with
1191 // overloaded return type, in which case the exact struct type will be
1192 // mangled into the name.
1193 SmallVector<Intrinsic::IITDescriptor> Desc;
1194 Intrinsic::getIntrinsicInfoTableEntries(F->getIntrinsicID(), Desc);
1195 if (Desc.front().Kind == Intrinsic::IITDescriptor::Struct) {
1196 auto *FT = F->getFunctionType();
1197 auto *NewST = StructType::get(ST->getContext(), ST->elements());
1198 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1199 std::string Name = F->getName().str();
1200 rename(F);
1201 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
1202 Name, F->getParent());
1203
1204 // The new function may also need remangling.
1205 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
1206 NewFn = *Result;
1207 return true;
1208 }
1209 }
1210
1211 // Remangle our intrinsic since we upgrade the mangling
1212 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
1213 if (Result != std::nullopt) {
1214 NewFn = *Result;
1215 return true;
1216 }
1217
1218 // This may not belong here. This function is effectively being overloaded
1219 // to both detect an intrinsic which needs upgrading, and to provide the
1220 // upgraded form of the intrinsic. We should perhaps have two separate
1221 // functions for this.
1222 return false;
1223}
1224
1225bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
1226 NewFn = nullptr;
1227 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
1228 assert(F != NewFn && "Intrinsic function upgraded to the same function")(static_cast <bool> (F != NewFn && "Intrinsic function upgraded to the same function"
) ? void (0) : __assert_fail ("F != NewFn && \"Intrinsic function upgraded to the same function\""
, "llvm/lib/IR/AutoUpgrade.cpp", 1228, __extension__ __PRETTY_FUNCTION__
));
1229
1230 // Upgrade intrinsic attributes. This does not change the function.
1231 if (NewFn)
1232 F = NewFn;
1233 if (Intrinsic::ID id = F->getIntrinsicID())
1234 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
1235 return Upgraded;
1236}
1237
1238GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
1239 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
1240 GV->getName() == "llvm.global_dtors")) ||
1241 !GV->hasInitializer())
1242 return nullptr;
1243 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
1244 if (!ATy)
1245 return nullptr;
1246 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
1247 if (!STy || STy->getNumElements() != 2)
1248 return nullptr;
1249
1250 LLVMContext &C = GV->getContext();
1251 IRBuilder<> IRB(C);
1252 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
1253 IRB.getInt8PtrTy());
1254 Constant *Init = GV->getInitializer();
1255 unsigned N = Init->getNumOperands();
1256 std::vector<Constant *> NewCtors(N);
1257 for (unsigned i = 0; i != N; ++i) {
1258 auto Ctor = cast<Constant>(Init->getOperand(i));
1259 NewCtors[i] = ConstantStruct::get(
1260 EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
1261 Constant::getNullValue(IRB.getInt8PtrTy()));
1262 }
1263 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
1264
1265 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
1266 NewInit, GV->getName());
1267}
1268
1269// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
1270// to byte shuffles.
1271static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
1272 Value *Op, unsigned Shift) {
1273 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1274 unsigned NumElts = ResultTy->getNumElements() * 8;
1275
1276 // Bitcast from a 64-bit element type to a byte element type.
1277 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1278 Op = Builder.CreateBitCast(Op, VecTy, "cast");
1279
1280 // We'll be shuffling in zeroes.
1281 Value *Res = Constant::getNullValue(VecTy);
1282
1283 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1284 // we'll just return the zero vector.
1285 if (Shift < 16) {
1286 int Idxs[64];
1287 // 256/512-bit version is split into 2/4 16-byte lanes.
1288 for (unsigned l = 0; l != NumElts; l += 16)
1289 for (unsigned i = 0; i != 16; ++i) {
1290 unsigned Idx = NumElts + i - Shift;
1291 if (Idx < NumElts)
1292 Idx -= NumElts - 16; // end of lane, switch operand.
1293 Idxs[l + i] = Idx + l;
1294 }
1295
1296 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
1297 }
1298
1299 // Bitcast back to a 64-bit element type.
1300 return Builder.CreateBitCast(Res, ResultTy, "cast");
1301}
1302
1303// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
1304// to byte shuffles.
1305static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
1306 unsigned Shift) {
1307 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1308 unsigned NumElts = ResultTy->getNumElements() * 8;
1309
1310 // Bitcast from a 64-bit element type to a byte element type.
1311 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1312 Op = Builder.CreateBitCast(Op, VecTy, "cast");
1313
1314 // We'll be shuffling in zeroes.
1315 Value *Res = Constant::getNullValue(VecTy);
1316
1317 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1318 // we'll just return the zero vector.
1319 if (Shift < 16) {
1320 int Idxs[64];
1321 // 256/512-bit version is split into 2/4 16-byte lanes.
1322 for (unsigned l = 0; l != NumElts; l += 16)
1323 for (unsigned i = 0; i != 16; ++i) {
1324 unsigned Idx = i + Shift;
1325 if (Idx >= 16)
1326 Idx += NumElts - 16; // end of lane, switch operand.
1327 Idxs[l + i] = Idx + l;
1328 }
1329
1330 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
1331 }
1332
1333 // Bitcast back to a 64-bit element type.
1334 return Builder.CreateBitCast(Res, ResultTy, "cast");
1335}
1336
1337static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
1338 unsigned NumElts) {
1339 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements")(static_cast <bool> (isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements"
) ? void (0) : __assert_fail ("isPowerOf2_32(NumElts) && \"Expected power-of-2 mask elements\""
, "llvm/lib/IR/AutoUpgrade.cpp", 1339, __extension__ __PRETTY_FUNCTION__
));
1340 llvm::VectorType *MaskTy = FixedVectorType::get(
1341 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
1342 Mask = Builder.CreateBitCast(Mask, MaskTy);
1343
1344 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
1345 // i8 and we need to extract down to the right number of elements.
1346 if (NumElts <= 4) {
1347 int Indices[4];
1348 for (unsigned i = 0; i != NumElts; ++i)
1349 Indices[i] = i;
1350 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
1351 "extract");
1352 }
1353
1354 return Mask;
1355}
1356
1357static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
1358 Value *Op0, Value *Op1) {
1359 // If the mask is all ones just emit the first operation.
1360 if (const auto *C = dyn_cast<Constant>(Mask))
1361 if (C->isAllOnesValue())
1362 return Op0;
1363
1364 Mask = getX86MaskVec(Builder, Mask,
1365 cast<FixedVectorType>(Op0->getType())->getNumElements());
1366 return Builder.CreateSelect(Mask, Op0, Op1);
1367}
1368
1369static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask,
1370 Value *Op0, Value *Op1) {
1371 // If the mask is all ones just emit the first operation.
1372 if (const auto *C = dyn_cast<Constant>(Mask))
1373 if (C->isAllOnesValue())
1374 return Op0;
1375
1376 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
1377 Mask->getType()->getIntegerBitWidth());
1378 Mask = Builder.CreateBitCast(Mask, MaskTy);
1379 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
1380 return Builder.CreateSelect(Mask, Op0, Op1);
1381}
1382
1383// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
1384// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
1385// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
1386static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
1387 Value *Op1, Value *Shift,
1388 Value *Passthru, Value *Mask,
1389 bool IsVALIGN) {
1390 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1391
1392 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1393 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!")(static_cast <bool> ((IsVALIGN || NumElts % 16 == 0) &&
"Illegal NumElts for PALIGNR!") ? void (0) : __assert_fail (
"(IsVALIGN || NumElts % 16 == 0) && \"Illegal NumElts for PALIGNR!\""
, "llvm/lib/IR/AutoUpgrade.cpp", 1393, __extension__ __PRETTY_FUNCTION__
));
1394 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!")(static_cast <bool> ((!IsVALIGN || NumElts <= 16) &&
"NumElts too large for VALIGN!") ? void (0) : __assert_fail (
"(!IsVALIGN || NumElts <= 16) && \"NumElts too large for VALIGN!\""
, "llvm/lib/IR/AutoUpgrade.cpp", 1394, __extension__ __PRETTY_FUNCTION__
));
1395 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!")(static_cast <bool> (isPowerOf2_32(NumElts) && "NumElts not a power of 2!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NumElts) && \"NumElts not a power of 2!\""
, "llvm/lib/IR/AutoUpgrade.cpp", 1395, __extension__ __PRETTY_FUNCTION__
));
1396
1397 // Mask the immediate for VALIGN.
1398 if (IsVALIGN)
1399 ShiftVal &= (NumElts - 1);
1400
1401 // If palignr is shifting the pair of vectors more than the size of two
1402 // lanes, emit zero.
1403 if (ShiftVal >= 32)
1404 return llvm::Constant::getNullValue(Op0->getType());
1405
1406 // If palignr is shifting the pair of input vectors more than one lane,
1407 // but less than two lanes, convert to shifting in zeroes.
1408 if (ShiftVal > 16) {
1409 ShiftVal -= 16;
1410 Op1 = Op0;
1411 Op0 = llvm::Constant::getNullValue(Op0->getType());
1412 }
1413
1414 int Indices[64];
1415 // 256-bit palignr operates on 128-bit lanes so we need to handle that
1416 for (unsigned l = 0; l < NumElts; l += 16) {
1417 for (unsigned i = 0; i != 16; ++i) {
1418 unsigned Idx = ShiftVal + i;
1419 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
1420 Idx += NumElts - 16; // End of lane, switch operand.
1421 Indices[l + i] = Idx + l;
1422 }
1423 }
1424
1425 Value *Align = Builder.CreateShuffleVector(
1426 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
1427
1428 return EmitX86Select(Builder, Mask, Align, Passthru);
1429}
1430
1431static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
1432 bool ZeroMask, bool IndexForm) {
1433 Type *Ty = CI.getType();
1434 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
1435 unsigned EltWidth = Ty->getScalarSizeInBits();
1436 bool IsFloat = Ty->isFPOrFPVectorTy();
1437 Intrinsic::ID IID;
1438 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1439 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1440 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1441 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1442 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1443 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1444 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1445 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1446 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1447 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1448 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1449 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1450 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1451 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1452 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1453 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1454 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1455 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1456 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1457 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1458 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1459 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1460 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1461 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1462 else if (VecWidth == 128 && EltWidth == 16)
1463 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1464 else if (VecWidth == 256 && EltWidth == 16)
1465 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1466 else if (VecWidth == 512 && EltWidth == 16)
1467 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1468 else if (VecWidth == 128 && EltWidth == 8)
1469 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1470 else if (VecWidth == 256 && EltWidth == 8)
1471 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1472 else if (VecWidth == 512 && EltWidth == 8)
1473 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1474 else
1475 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1475);
1476
1477 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
1478 CI.getArgOperand(2) };
1479
1480 // If this isn't index form we need to swap operand 0 and 1.
1481 if (!IndexForm)
1482 std::swap(Args[0], Args[1]);
1483
1484 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1485 Args);
1486 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
1487 : Builder.CreateBitCast(CI.getArgOperand(1),
1488 Ty);
1489 return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
1490}
1491
1492static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
1493 Intrinsic::ID IID) {
1494 Type *Ty = CI.getType();
1495 Value *Op0 = CI.getOperand(0);
1496 Value *Op1 = CI.getOperand(1);
1497 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1498 Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
1499
1500 if (CI.arg_size() == 4) { // For masked intrinsics.
1501 Value *VecSrc = CI.getOperand(2);
1502 Value *Mask = CI.getOperand(3);
1503 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1504 }
1505 return Res;
1506}
1507
1508static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
1509 bool IsRotateRight) {
1510 Type *Ty = CI.getType();
1511 Value *Src = CI.getArgOperand(0);
1512 Value *Amt = CI.getArgOperand(1);
1513
1514 // Amount may be scalar immediate, in which case create a splat vector.
1515 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1516 // we only care about the lowest log2 bits anyway.
1517 if (Amt->getType() != Ty) {
1518 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1519 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1520 Amt = Builder.CreateVectorSplat(NumElts, Amt);
1521 }
1522
1523 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1524 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1525 Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
1526
1527 if (CI.arg_size() == 4) { // For masked intrinsics.
1528 Value *VecSrc = CI.getOperand(2);
1529 Value *Mask = CI.getOperand(3);
1530 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1531 }
1532 return Res;
1533}
1534
1535static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
1536 bool IsSigned) {
1537 Type *Ty = CI.getType();
1538 Value *LHS = CI.getArgOperand(0);
1539 Value *RHS = CI.getArgOperand(1);
1540
1541 CmpInst::Predicate Pred;
1542 switch (Imm) {
1543 case 0x0:
1544 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1545 break;
1546 case 0x1:
1547 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1548 break;
1549 case 0x2:
1550 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1551 break;
1552 case 0x3:
1553 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1554 break;
1555 case 0x4:
1556 Pred = ICmpInst::ICMP_EQ;
1557 break;
1558 case 0x5:
1559 Pred = ICmpInst::ICMP_NE;
1560 break;
1561 case 0x6:
1562 return Constant::getNullValue(Ty); // FALSE
1563 case 0x7:
1564 return Constant::getAllOnesValue(Ty); // TRUE
1565 default:
1566 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate")::llvm::llvm_unreachable_internal("Unknown XOP vpcom/vpcomu predicate"
, "llvm/lib/IR/AutoUpgrade.cpp", 1566);
1567 }
1568
1569 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
1570 Value *Ext = Builder.CreateSExt(Cmp, Ty);
1571 return Ext;
1572}
1573
1574static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
1575 bool IsShiftRight, bool ZeroMask) {
1576 Type *Ty = CI.getType();
1577 Value *Op0 = CI.getArgOperand(0);
1578 Value *Op1 = CI.getArgOperand(1);
1579 Value *Amt = CI.getArgOperand(2);
1580
1581 if (IsShiftRight)
1582 std::swap(Op0, Op1);
1583
1584 // Amount may be scalar immediate, in which case create a splat vector.
1585 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1586 // we only care about the lowest log2 bits anyway.
1587 if (Amt->getType() != Ty) {
1588 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1589 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1590 Amt = Builder.CreateVectorSplat(NumElts, Amt);
1591 }
1592
1593 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1594 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1595 Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
1596
1597 unsigned NumArgs = CI.arg_size();
1598 if (NumArgs >= 4) { // For masked intrinsics.
1599 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
1600 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
1601 CI.getArgOperand(0);
1602 Value *Mask = CI.getOperand(NumArgs - 1);
1603 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1604 }
1605 return Res;
1606}
1607
1608static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
1609 Value *Ptr, Value *Data, Value *Mask,
1610 bool Aligned) {
1611 // Cast the pointer to the right type.
1612 Ptr = Builder.CreateBitCast(Ptr,
1613 llvm::PointerType::getUnqual(Data->getType()));
1614 const Align Alignment =
1615 Aligned
1616 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
1617 : Align(1);
1618
1619 // If the mask is all ones just emit a regular store.
1620 if (const auto *C = dyn_cast<Constant>(Mask))
1621 if (C->isAllOnesValue())
1622 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
1623
1624 // Convert the mask from an integer type to a vector of i1.
1625 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
1626 Mask = getX86MaskVec(Builder, Mask, NumElts);
1627 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
1628}
1629
1630static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
1631 Value *Ptr, Value *Passthru, Value *Mask,
1632 bool Aligned) {
1633 Type *ValTy = Passthru->getType();
1634 // Cast the pointer to the right type.
1635 Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
1636 const Align Alignment =
1637 Aligned
1638 ? Align(
1639 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
1640 8)
1641 : Align(1);
1642
1643 // If the mask is all ones just emit a regular store.
1644 if (const auto *C = dyn_cast<Constant>(Mask))
1645 if (C->isAllOnesValue())
1646 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
1647
1648 // Convert the mask from an integer type to a vector of i1.
1649 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
1650 Mask = getX86MaskVec(Builder, Mask, NumElts);
1651 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
1652}
1653
1654static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
1655 Type *Ty = CI.getType();
1656 Value *Op0 = CI.getArgOperand(0);
1657 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);
1658 Value *Res = Builder.CreateCall(F, {Op0, Builder.getInt1(false)});
1659 if (CI.arg_size() == 3)
1660 Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
1661 return Res;
1662}
1663
1664static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
1665 Type *Ty = CI.getType();
1666
1667 // Arguments have a vXi32 type so cast to vXi64.
1668 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
1669 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
1670
1671 if (IsSigned) {
1672 // Shift left then arithmetic shift right.
1673 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1674 LHS = Builder.CreateShl(LHS, ShiftAmt);
1675 LHS = Builder.CreateAShr(LHS, ShiftAmt);
1676 RHS = Builder.CreateShl(RHS, ShiftAmt);
1677 RHS = Builder.CreateAShr(RHS, ShiftAmt);
1678 } else {
1679 // Clear the upper bits.
1680 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1681 LHS = Builder.CreateAnd(LHS, Mask);
1682 RHS = Builder.CreateAnd(RHS, Mask);
1683 }
1684
1685 Value *Res = Builder.CreateMul(LHS, RHS);
1686
1687 if (CI.arg_size() == 4)
1688 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
1689
1690 return Res;
1691}
1692
1693// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
1694static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
1695 Value *Mask) {
1696 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
7
←
The object is a 'CastReturnType'
→
8
←
'NumElts' initialized here
→
1697 if (Mask
8.1
'Mask' is null
) {
9
←
Taking false branch
→
1698 const auto *C = dyn_cast<Constant>(Mask);
1699 if (!C || !C->isAllOnesValue())
1700 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
1701 }
1702
1703 if (NumElts < 8) {
10
←
Assuming 'NumElts' is < 8
→
11
←
Taking true branch
→
1704 int Indices[8];
1705 for (unsigned i = 0; i != NumElts; ++i)
12
←
Assuming 'i' is equal to 'NumElts'
→
13
←
Loop condition is false. Execution continues on line 1707
→
1706 Indices[i] = i;
1707 for (unsigned i = NumElts; i != 8; ++i)
14
←
Loop condition is true. Entering loop body
→
1708 Indices[i] = NumElts + i % NumElts;
15
←
Division by zero
1709 Vec = Builder.CreateShuffleVector(Vec,
1710 Constant::getNullValue(Vec->getType()),
1711 Indices);
1712 }
1713 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
1714}
1715
1716static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
1717 unsigned CC, bool Signed) {
1718 Value *Op0 = CI.getArgOperand(0);
1719 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1
The object is a 'CastReturnType'
→
1720
1721 Value *Cmp;
1722 if (CC == 3) {
2
←
Assuming 'CC' is not equal to 3
→
3
←
Taking false branch
→
1723 Cmp = Constant::getNullValue(
1724 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1725 } else if (CC == 7) {
4
←
Assuming 'CC' is equal to 7
→
5
←
Taking true branch
→
1726 Cmp = Constant::getAllOnesValue(
1727 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1728 } else {
1729 ICmpInst::Predicate Pred;
1730 switch (CC) {
1731 default: llvm_unreachable("Unknown condition code")::llvm::llvm_unreachable_internal("Unknown condition code", "llvm/lib/IR/AutoUpgrade.cpp"
, 1731);
1732 case 0: Pred = ICmpInst::ICMP_EQ; break;
1733 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
1734 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
1735 case 4: Pred = ICmpInst::ICMP_NE; break;
1736 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
1737 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
1738 }
1739 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
1740 }
1741
1742 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
1743
1744 return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask);
6
←
Calling 'ApplyX86MaskOn1BitsVec'
→
1745}
1746
1747// Replace a masked intrinsic with an older unmasked intrinsic.
1748static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
1749 Intrinsic::ID IID) {
1750 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
1751 Value *Rep = Builder.CreateCall(Intrin,
1752 { CI.getArgOperand(0), CI.getArgOperand(1) });
1753 return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
1754}
1755
1756static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
1757 Value* A = CI.getArgOperand(0);
1758 Value* B = CI.getArgOperand(1);
1759 Value* Src = CI.getArgOperand(2);
1760 Value* Mask = CI.getArgOperand(3);
1761
1762 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
1763 Value* Cmp = Builder.CreateIsNotNull(AndNode);
1764 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
1765 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
1766 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
1767 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
1768}
1769
1770
1771static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
1772 Value* Op = CI.getArgOperand(0);
1773 Type* ReturnOp = CI.getType();
1774 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
1775 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
1776 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
1777}
1778
1779// Replace intrinsic with unmasked version and a select.
1780static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
1781 CallBase &CI, Value *&Rep) {
1782 Name = Name.substr(12); // Remove avx512.mask.
1783
1784 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
1785 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
1786 Intrinsic::ID IID;
1787 if (Name.startswith("max.p")) {
1788 if (VecWidth == 128 && EltWidth == 32)
1789 IID = Intrinsic::x86_sse_max_ps;
1790 else if (VecWidth == 128 && EltWidth == 64)
1791 IID = Intrinsic::x86_sse2_max_pd;
1792 else if (VecWidth == 256 && EltWidth == 32)
1793 IID = Intrinsic::x86_avx_max_ps_256;
1794 else if (VecWidth == 256 && EltWidth == 64)
1795 IID = Intrinsic::x86_avx_max_pd_256;
1796 else
1797 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1797);
1798 } else if (Name.startswith("min.p")) {
1799 if (VecWidth == 128 && EltWidth == 32)
1800 IID = Intrinsic::x86_sse_min_ps;
1801 else if (VecWidth == 128 && EltWidth == 64)
1802 IID = Intrinsic::x86_sse2_min_pd;
1803 else if (VecWidth == 256 && EltWidth == 32)
1804 IID = Intrinsic::x86_avx_min_ps_256;
1805 else if (VecWidth == 256 && EltWidth == 64)
1806 IID = Intrinsic::x86_avx_min_pd_256;
1807 else
1808 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1808);
1809 } else if (Name.startswith("pshuf.b.")) {
1810 if (VecWidth == 128)
1811 IID = Intrinsic::x86_ssse3_pshuf_b_128;
1812 else if (VecWidth == 256)
1813 IID = Intrinsic::x86_avx2_pshuf_b;
1814 else if (VecWidth == 512)
1815 IID = Intrinsic::x86_avx512_pshuf_b_512;
1816 else
1817 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1817);
1818 } else if (Name.startswith("pmul.hr.sw.")) {
1819 if (VecWidth == 128)
1820 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
1821 else if (VecWidth == 256)
1822 IID = Intrinsic::x86_avx2_pmul_hr_sw;
1823 else if (VecWidth == 512)
1824 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
1825 else
1826 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1826);
1827 } else if (Name.startswith("pmulh.w.")) {
1828 if (VecWidth == 128)
1829 IID = Intrinsic::x86_sse2_pmulh_w;
1830 else if (VecWidth == 256)
1831 IID = Intrinsic::x86_avx2_pmulh_w;
1832 else if (VecWidth == 512)
1833 IID = Intrinsic::x86_avx512_pmulh_w_512;
1834 else
1835 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1835);
1836 } else if (Name.startswith("pmulhu.w.")) {
1837 if (VecWidth == 128)
1838 IID = Intrinsic::x86_sse2_pmulhu_w;
1839 else if (VecWidth == 256)
1840 IID = Intrinsic::x86_avx2_pmulhu_w;
1841 else if (VecWidth == 512)
1842 IID = Intrinsic::x86_avx512_pmulhu_w_512;
1843 else
1844 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1844);
1845 } else if (Name.startswith("pmaddw.d.")) {
1846 if (VecWidth == 128)
1847 IID = Intrinsic::x86_sse2_pmadd_wd;
1848 else if (VecWidth == 256)
1849 IID = Intrinsic::x86_avx2_pmadd_wd;
1850 else if (VecWidth == 512)
1851 IID = Intrinsic::x86_avx512_pmaddw_d_512;
1852 else
1853 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1853);
1854 } else if (Name.startswith("pmaddubs.w.")) {
1855 if (VecWidth == 128)
1856 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
1857 else if (VecWidth == 256)
1858 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
1859 else if (VecWidth == 512)
1860 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
1861 else
1862 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1862);
1863 } else if (Name.startswith("packsswb.")) {
1864 if (VecWidth == 128)
1865 IID = Intrinsic::x86_sse2_packsswb_128;
1866 else if (VecWidth == 256)
1867 IID = Intrinsic::x86_avx2_packsswb;
1868 else if (VecWidth == 512)
1869 IID = Intrinsic::x86_avx512_packsswb_512;
1870 else
1871 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1871);
1872 } else if (Name.startswith("packssdw.")) {
1873 if (VecWidth == 128)
1874 IID = Intrinsic::x86_sse2_packssdw_128;
1875 else if (VecWidth == 256)
1876 IID = Intrinsic::x86_avx2_packssdw;
1877 else if (VecWidth == 512)
1878 IID = Intrinsic::x86_avx512_packssdw_512;
1879 else
1880 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1880);
1881 } else if (Name.startswith("packuswb.")) {
1882 if (VecWidth == 128)
1883 IID = Intrinsic::x86_sse2_packuswb_128;
1884 else if (VecWidth == 256)
1885 IID = Intrinsic::x86_avx2_packuswb;
1886 else if (VecWidth == 512)
1887 IID = Intrinsic::x86_avx512_packuswb_512;
1888 else
1889 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1889);
1890 } else if (Name.startswith("packusdw.")) {
1891 if (VecWidth == 128)
1892 IID = Intrinsic::x86_sse41_packusdw;
1893 else if (VecWidth == 256)
1894 IID = Intrinsic::x86_avx2_packusdw;
1895 else if (VecWidth == 512)
1896 IID = Intrinsic::x86_avx512_packusdw_512;
1897 else
1898 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1898);
1899 } else if (Name.startswith("vpermilvar.")) {
1900 if (VecWidth == 128 && EltWidth == 32)
1901 IID = Intrinsic::x86_avx_vpermilvar_ps;
1902 else if (VecWidth == 128 && EltWidth == 64)
1903 IID = Intrinsic::x86_avx_vpermilvar_pd;
1904 else if (VecWidth == 256 && EltWidth == 32)
1905 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
1906 else if (VecWidth == 256 && EltWidth == 64)
1907 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
1908 else if (VecWidth == 512 && EltWidth == 32)
1909 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
1910 else if (VecWidth == 512 && EltWidth == 64)
1911 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
1912 else
1913 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1913);
1914 } else if (Name == "cvtpd2dq.256") {
1915 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
1916 } else if (Name == "cvtpd2ps.256") {
1917 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
1918 } else if (Name == "cvttpd2dq.256") {
1919 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
1920 } else if (Name == "cvttps2dq.128") {
1921 IID = Intrinsic::x86_sse2_cvttps2dq;
1922 } else if (Name == "cvttps2dq.256") {
1923 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
1924 } else if (Name.startswith("permvar.")) {
1925 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
1926 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1927 IID = Intrinsic::x86_avx2_permps;
1928 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1929 IID = Intrinsic::x86_avx2_permd;
1930 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1931 IID = Intrinsic::x86_avx512_permvar_df_256;
1932 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1933 IID = Intrinsic::x86_avx512_permvar_di_256;
1934 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1935 IID = Intrinsic::x86_avx512_permvar_sf_512;
1936 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1937 IID = Intrinsic::x86_avx512_permvar_si_512;
1938 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1939 IID = Intrinsic::x86_avx512_permvar_df_512;
1940 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1941 IID = Intrinsic::x86_avx512_permvar_di_512;
1942 else if (VecWidth == 128 && EltWidth == 16)
1943 IID = Intrinsic::x86_avx512_permvar_hi_128;
1944 else if (VecWidth == 256 && EltWidth == 16)
1945 IID = Intrinsic::x86_avx512_permvar_hi_256;
1946 else if (VecWidth == 512 && EltWidth == 16)
1947 IID = Intrinsic::x86_avx512_permvar_hi_512;
1948 else if (VecWidth == 128 && EltWidth == 8)
1949 IID = Intrinsic::x86_avx512_permvar_qi_128;
1950 else if (VecWidth == 256 && EltWidth == 8)
1951 IID = Intrinsic::x86_avx512_permvar_qi_256;
1952 else if (VecWidth == 512 && EltWidth == 8)
1953 IID = Intrinsic::x86_avx512_permvar_qi_512;
1954 else
1955 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1955);
1956 } else if (Name.startswith("dbpsadbw.")) {
1957 if (VecWidth == 128)
1958 IID = Intrinsic::x86_avx512_dbpsadbw_128;
1959 else if (VecWidth == 256)
1960 IID = Intrinsic::x86_avx512_dbpsadbw_256;
1961 else if (VecWidth == 512)
1962 IID = Intrinsic::x86_avx512_dbpsadbw_512;
1963 else
1964 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1964);
1965 } else if (Name.startswith("pmultishift.qb.")) {
1966 if (VecWidth == 128)
1967 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
1968 else if (VecWidth == 256)
1969 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
1970 else if (VecWidth == 512)
1971 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
1972 else
1973 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1973);
1974 } else if (Name.startswith("conflict.")) {
1975 if (Name[9] == 'd' && VecWidth == 128)
1976 IID = Intrinsic::x86_avx512_conflict_d_128;
1977 else if (Name[9] == 'd' && VecWidth == 256)
1978 IID = Intrinsic::x86_avx512_conflict_d_256;
1979 else if (Name[9] == 'd' && VecWidth == 512)
1980 IID = Intrinsic::x86_avx512_conflict_d_512;
1981 else if (Name[9] == 'q' && VecWidth == 128)
1982 IID = Intrinsic::x86_avx512_conflict_q_128;
1983 else if (Name[9] == 'q' && VecWidth == 256)
1984 IID = Intrinsic::x86_avx512_conflict_q_256;
1985 else if (Name[9] == 'q' && VecWidth == 512)
1986 IID = Intrinsic::x86_avx512_conflict_q_512;
1987 else
1988 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 1988);
1989 } else if (Name.startswith("pavg.")) {
1990 if (Name[5] == 'b' && VecWidth == 128)
1991 IID = Intrinsic::x86_sse2_pavg_b;
1992 else if (Name[5] == 'b' && VecWidth == 256)
1993 IID = Intrinsic::x86_avx2_pavg_b;
1994 else if (Name[5] == 'b' && VecWidth == 512)
1995 IID = Intrinsic::x86_avx512_pavg_b_512;
1996 else if (Name[5] == 'w' && VecWidth == 128)
1997 IID = Intrinsic::x86_sse2_pavg_w;
1998 else if (Name[5] == 'w' && VecWidth == 256)
1999 IID = Intrinsic::x86_avx2_pavg_w;
2000 else if (Name[5] == 'w' && VecWidth == 512)
2001 IID = Intrinsic::x86_avx512_pavg_w_512;
2002 else
2003 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 2003);
2004 } else
2005 return false;
2006
2007 SmallVector<Value *, 4> Args(CI.args());
2008 Args.pop_back();
2009 Args.pop_back();
2010 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
2011 Args);
2012 unsigned NumArgs = CI.arg_size();
2013 Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
2014 CI.getArgOperand(NumArgs - 2));
2015 return true;
2016}
2017
2018/// Upgrade comment in call to inline asm that represents an objc retain release
2019/// marker.
2020void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
2021 size_t Pos;
2022 if (AsmStr->find("mov\tfp") == 0 &&
2023 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
2024 (Pos = AsmStr->find("# marker")) != std::string::npos) {
2025 AsmStr->replace(Pos, 1, ";");
2026 }
2027}
2028
2029static Value *UpgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
2030 IRBuilder<> &Builder) {
2031 if (Name == "mve.vctp64.old") {
2032 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
2033 // correct type.
2034 Value *VCTP = Builder.CreateCall(
2035 Intrinsic::getDeclaration(F->getParent(), Intrinsic::arm_mve_vctp64),
2036 CI->getArgOperand(0), CI->getName());
2037 Value *C1 = Builder.CreateCall(
2038 Intrinsic::getDeclaration(
2039 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2040 {VectorType::get(Builder.getInt1Ty(), 2, false)}),
2041 VCTP);
2042 return Builder.CreateCall(
2043 Intrinsic::getDeclaration(
2044 F->getParent(), Intrinsic::arm_mve_pred_i2v,
2045 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2046 C1);
2047 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
2048 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
2049 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
2050 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
2051 Name ==
2052 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
2053 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
2054 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
2055 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
2056 Name ==
2057 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
2058 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
2059 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
2060 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
2061 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
2062 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
2063 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
2064 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
2065 std::vector<Type *> Tys;
2066 unsigned ID = CI->getIntrinsicID();
2067 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
2068 switch (ID) {
2069 case Intrinsic::arm_mve_mull_int_predicated:
2070 case Intrinsic::arm_mve_vqdmull_predicated:
2071 case Intrinsic::arm_mve_vldr_gather_base_predicated:
2072 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
2073 break;
2074 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
2075 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
2076 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
2077 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
2078 V2I1Ty};
2079 break;
2080 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
2081 Tys = {CI->getType(), CI->getOperand(0)->getType(),
2082 CI->getOperand(1)->getType(), V2I1Ty};
2083 break;
2084 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
2085 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
2086 CI->getOperand(2)->getType(), V2I1Ty};
2087 break;
2088 case Intrinsic::arm_cde_vcx1q_predicated:
2089 case Intrinsic::arm_cde_vcx1qa_predicated:
2090 case Intrinsic::arm_cde_vcx2q_predicated:
2091 case Intrinsic::arm_cde_vcx2qa_predicated:
2092 case Intrinsic::arm_cde_vcx3q_predicated:
2093 case Intrinsic::arm_cde_vcx3qa_predicated:
2094 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
2095 break;
2096 default:
2097 llvm_unreachable("Unhandled Intrinsic!")::llvm::llvm_unreachable_internal("Unhandled Intrinsic!", "llvm/lib/IR/AutoUpgrade.cpp"
, 2097);
2098 }
2099
2100 std::vector<Value *> Ops;
2101 for (Value *Op : CI->args()) {
2102 Type *Ty = Op->getType();
2103 if (Ty->getScalarSizeInBits() == 1) {
2104 Value *C1 = Builder.CreateCall(
2105 Intrinsic::getDeclaration(
2106 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2107 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2108 Op);
2109 Op = Builder.CreateCall(
2110 Intrinsic::getDeclaration(F->getParent(),
2111 Intrinsic::arm_mve_pred_i2v, {V2I1Ty}),
2112 C1);
2113 }
2114 Ops.push_back(Op);
2115 }
2116
2117 Function *Fn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
2118 return Builder.CreateCall(Fn, Ops, CI->getName());
2119 }
2120 llvm_unreachable("Unknown function for ARM CallBase upgrade.")::llvm::llvm_unreachable_internal("Unknown function for ARM CallBase upgrade."
, "llvm/lib/IR/AutoUpgrade.cpp", 2120);
2121}
2122
2123/// Upgrade a call to an old intrinsic. All argument and return casting must be
2124/// provided to seamlessly integrate with existing context.
2125void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
2126 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
2127 // checks the callee's function type matches. It's likely we need to handle
2128 // type changes here.
2129 Function *F = dyn_cast<Function>(CI->getCalledOperand());
2130 if (!F)
2131 return;
2132
2133 LLVMContext &C = CI->getContext();
2134 IRBuilder<> Builder(C);
2135 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
2136
2137 if (!NewFn) {
2138 // Get the Function's name.
2139 StringRef Name = F->getName();
2140
2141 assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'")(static_cast <bool> (Name.startswith("llvm.") &&
"Intrinsic doesn't start with 'llvm.'") ? void (0) : __assert_fail
("Name.startswith(\"llvm.\") && \"Intrinsic doesn't start with 'llvm.'\""
, "llvm/lib/IR/AutoUpgrade.cpp", 2141, __extension__ __PRETTY_FUNCTION__
));
2142 Name = Name.substr(5);
2143
2144 bool IsX86 = Name.startswith("x86.");
2145 if (IsX86)
2146 Name = Name.substr(4);
2147 bool IsNVVM = Name.startswith("nvvm.");
2148 if (IsNVVM)
2149 Name = Name.substr(5);
2150 bool IsARM = Name.startswith("arm.");
2151 if (IsARM)
2152 Name = Name.substr(4);
2153
2154 if (IsX86 && Name.startswith("sse4a.movnt.")) {
2155 Module *M = F->getParent();
2156 SmallVector<Metadata *, 1> Elts;
2157 Elts.push_back(
2158 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
2159 MDNode *Node = MDNode::get(C, Elts);
2160
2161 Value *Arg0 = CI->getArgOperand(0);
2162 Value *Arg1 = CI->getArgOperand(1);
2163
2164 // Nontemporal (unaligned) store of the 0'th element of the float/double
2165 // vector.
2166 Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();
2167 PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
2168 Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");
2169 Value *Extract =
2170 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
2171
2172 StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1));
2173 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
2174
2175 // Remove intrinsic.
2176 CI->eraseFromParent();
2177 return;
2178 }
2179
2180 if (IsX86 && (Name.startswith("avx.movnt.") ||
2181 Name.startswith("avx512.storent."))) {
2182 Module *M = F->getParent();
2183 SmallVector<Metadata *, 1> Elts;
2184 Elts.push_back(
2185 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
2186 MDNode *Node = MDNode::get(C, Elts);
2187
2188 Value *Arg0 = CI->getArgOperand(0);
2189 Value *Arg1 = CI->getArgOperand(1);
2190
2191 // Convert the type of the pointer to a pointer to the stored type.
2192 Value *BC = Builder.CreateBitCast(Arg0,
2193 PointerType::getUnqual(Arg1->getType()),
2194 "cast");
2195 StoreInst *SI = Builder.CreateAlignedStore(
2196 Arg1, BC,
2197 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
2198 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
2199
2200 // Remove intrinsic.
2201 CI->eraseFromParent();
2202 return;
2203 }
2204
2205 if (IsX86 && Name == "sse2.storel.dq") {
2206 Value *Arg0 = CI->getArgOperand(0);
2207 Value *Arg1 = CI->getArgOperand(1);
2208
2209 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
2210 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
2211 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
2212 Value *BC = Builder.CreateBitCast(Arg0,
2213 PointerType::getUnqual(Elt->getType()),
2214 "cast");
2215 Builder.CreateAlignedStore(Elt, BC, Align(1));
2216
2217 // Remove intrinsic.
2218 CI->eraseFromParent();
2219 return;
2220 }
2221
2222 if (IsX86 && (Name.startswith("sse.storeu.") ||
2223 Name.startswith("sse2.storeu.") ||
2224 Name.startswith("avx.storeu."))) {
2225 Value *Arg0 = CI->getArgOperand(0);
2226 Value *Arg1 = CI->getArgOperand(1);
2227
2228 Arg0 = Builder.CreateBitCast(Arg0,
2229 PointerType::getUnqual(Arg1->getType()),
2230 "cast");
2231 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
2232
2233 // Remove intrinsic.
2234 CI->eraseFromParent();
2235 return;
2236 }
2237
2238 if (IsX86 && Name == "avx512.mask.store.ss") {
2239 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
2240 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2241 Mask, false);
2242
2243 // Remove intrinsic.
2244 CI->eraseFromParent();
2245 return;
2246 }
2247
2248 if (IsX86 && (Name.startswith("avx512.mask.store"))) {
2249 // "avx512.mask.storeu." or "avx512.mask.store."
2250 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
2251 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2252 CI->getArgOperand(2), Aligned);
2253
2254 // Remove intrinsic.
2255 CI->eraseFromParent();
2256 return;
2257 }
2258
2259 Value *Rep;
2260 // Upgrade packed integer vector compare intrinsics to compare instructions.
2261 if (IsX86 && (Name.startswith("sse2.pcmp") ||
2262 Name.startswith("avx2.pcmp"))) {
2263 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
2264 bool CmpEq = Name[9] == 'e';
2265 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
2266 CI->getArgOperand(0), CI->getArgOperand(1));
2267 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
2268 } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) {
2269 Type *ExtTy = Type::getInt32Ty(C);
2270 if (CI->getOperand(0)->getType()->isIntegerTy(8))
2271 ExtTy = Type::getInt64Ty(C);
2272 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
2273 ExtTy->getPrimitiveSizeInBits();
2274 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
2275 Rep = Builder.CreateVectorSplat(NumElts, Rep);
2276 } else if (IsX86 && (Name == "sse.sqrt.ss" ||
2277 Name == "sse2.sqrt.sd")) {
2278 Value *Vec = CI->getArgOperand(0);
2279 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
2280 Function *Intr = Intrinsic::getDeclaration(F->getParent(),
2281 Intrinsic::sqrt, Elt0->getType());
2282 Elt0 = Builder.CreateCall(Intr, Elt0);
2283 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
2284 } else if (IsX86 && (Name.startswith("avx.sqrt.p") ||
2285 Name.startswith("sse2.sqrt.p") ||
2286 Name.startswith("sse.sqrt.p"))) {
2287 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
2288 Intrinsic::sqrt,
2289 CI->getType()),
2290 {CI->getArgOperand(0)});
2291 } else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) {
2292 if (CI->arg_size() == 4 &&
2293 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2294 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2295 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
2296 : Intrinsic::x86_avx512_sqrt_pd_512;
2297
2298 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
2299 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
2300 IID), Args);
2301 } else {
2302 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
2303 Intrinsic::sqrt,
2304 CI->getType()),
2305 {CI->getArgOperand(0)});
2306 }
2307 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2308 CI->getArgOperand(1));
2309 } else if (IsX86 && (Name.startswith("avx512.ptestm") ||
2310 Name.startswith("avx512.ptestnm"))) {
2311 Value *Op0 = CI->getArgOperand(0);
2312 Value *Op1 = CI->getArgOperand(1);
2313 Value *Mask = CI->getArgOperand(2);
2314 Rep = Builder.CreateAnd(Op0, Op1);
2315 llvm::Type *Ty = Op0->getType();
2316 Value *Zero = llvm::Constant::getNullValue(Ty);
2317 ICmpInst::Predicate Pred =
2318 Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
2319 Rep = Builder.CreateICmp(Pred, Rep, Zero);
2320 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);
2321 } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
2322 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
2323 ->getNumElements();
2324 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
2325 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2326 CI->getArgOperand(1));
2327 } else if (IsX86 && (Name.startswith("avx512.kunpck"))) {
2328 unsigned NumElts = CI->getType()->getScalarSizeInBits();
2329 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
2330 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
2331 int Indices[64];
2332 for (unsigned i = 0; i != NumElts; ++i)
2333 Indices[i] = i;
2334
2335 // First extract half of each vector. This gives better codegen than
2336 // doing it in a single shuffle.
2337 LHS =
2338 Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
2339 RHS =
2340 Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
2341 // Concat the vectors.
2342 // NOTE: Operands have to be swapped to match intrinsic definition.
2343 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
2344 Rep = Builder.CreateBitCast(Rep, CI->getType());
2345 } else if (IsX86 && Name == "avx512.kand.w") {
2346 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2347 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2348 Rep = Builder.CreateAnd(LHS, RHS);
2349 Rep = Builder.CreateBitCast(Rep, CI->getType());
2350 } else if (IsX86 && Name == "avx512.kandn.w") {
2351 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2352 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2353 LHS = Builder.CreateNot(LHS);
2354 Rep = Builder.CreateAnd(LHS, RHS);
2355 Rep = Builder.CreateBitCast(Rep, CI->getType());
2356 } else if (IsX86 && Name == "avx512.kor.w") {
2357 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2358 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2359 Rep = Builder.CreateOr(LHS, RHS);
2360 Rep = Builder.CreateBitCast(Rep, CI->getType());
2361 } else if (IsX86 && Name == "avx512.kxor.w") {
2362 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2363 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2364 Rep = Builder.CreateXor(LHS, RHS);
2365 Rep = Builder.CreateBitCast(Rep, CI->getType());
2366 } else if (IsX86 && Name == "avx512.kxnor.w") {
2367 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2368 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2369 LHS = Builder.CreateNot(LHS);
2370 Rep = Builder.CreateXor(LHS, RHS);
2371 Rep = Builder.CreateBitCast(Rep, CI->getType());
2372 } else if (IsX86 && Name == "avx512.knot.w") {
2373 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2374 Rep = Builder.CreateNot(Rep);
2375 Rep = Builder.CreateBitCast(Rep, CI->getType());
2376 } else if (IsX86 &&
2377 (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w")) {
2378 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2379 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2380 Rep = Builder.CreateOr(LHS, RHS);
2381 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
2382 Value *C;
2383 if (Name[14] == 'c')
2384 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
2385 else
2386 C = ConstantInt::getNullValue(Builder.getInt16Ty());
2387 Rep = Builder.CreateICmpEQ(Rep, C);
2388 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
2389 } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
2390 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
2391 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
2392 Name == "sse.div.ss" || Name == "sse2.div.sd")) {
2393 Type *I32Ty = Type::getInt32Ty(C);
2394 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
2395 ConstantInt::get(I32Ty, 0));
2396 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
2397 ConstantInt::get(I32Ty, 0));
2398 Value *EltOp;
2399 if (Name.contains(".add."))
2400 EltOp = Builder.CreateFAdd(Elt0, Elt1);
2401 else if (Name.contains(".sub."))
2402 EltOp = Builder.CreateFSub(Elt0, Elt1);
2403 else if (Name.contains(".mul."))
2404 EltOp = Builder.CreateFMul(Elt0, Elt1);
2405 else
2406 EltOp = Builder.CreateFDiv(Elt0, Elt1);
2407 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
2408 ConstantInt::get(I32Ty, 0));
2409 } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) {
2410 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
2411 bool CmpEq = Name[16] == 'e';
2412 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
2413 } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) {
2414 Type *OpTy = CI->getArgOperand(0)->getType();
2415 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2416 Intrinsic::ID IID;
2417 switch (VecWidth) {
2418 default: llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 2418);
2419 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;
2420 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;
2421 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;
2422 }
2423
2424 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2425 { CI->getOperand(0), CI->getArgOperand(1) });
2426 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2427 } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) {
2428 Type *OpTy = CI->getArgOperand(0)->getType();
2429 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2430 unsigned EltWidth = OpTy->getScalarSizeInBits();
2431 Intrinsic::ID IID;
2432 if (VecWidth == 128 && EltWidth == 32)
2433 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2434 else if (VecWidth == 256 && EltWidth == 32)
2435 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2436 else if (VecWidth == 512 && EltWidth == 32)
2437 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2438 else if (VecWidth == 128 && EltWidth == 64)
2439 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2440 else if (VecWidth == 256 && EltWidth == 64)
2441 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2442 else if (VecWidth == 512 && EltWidth == 64)
2443 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2444 else
2445 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 2445);
2446
2447 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2448 { CI->getOperand(0), CI->getArgOperand(1) });
2449 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2450 } else if (IsX86 && Name.startswith("avx512.cmp.p")) {
2451 SmallVector<Value *, 4> Args(CI->args());
2452 Type *OpTy = Args[0]->getType();
2453 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2454 unsigned EltWidth = OpTy->getScalarSizeInBits();
2455 Intrinsic::ID IID;
2456 if (VecWidth == 128 && EltWidth == 32)
2457 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2458 else if (VecWidth == 256 && EltWidth == 32)
2459 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2460 else if (VecWidth == 512 && EltWidth == 32)
2461 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2462 else if (VecWidth == 128 && EltWidth == 64)
2463 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2464 else if (VecWidth == 256 && EltWidth == 64)
2465 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2466 else if (VecWidth == 512 && EltWidth == 64)
2467 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2468 else
2469 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 2469);
2470
2471 Value *Mask = Constant::getAllOnesValue(CI->getType());
2472 if (VecWidth == 512)
2473 std::swap(Mask, Args.back());
2474 Args.push_back(Mask);
2475
2476 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2477 Args);
2478 } else if (IsX86 && Name.startswith("avx512.mask.cmp.")) {
2479 // Integer compare intrinsics.
2480 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2481 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
2482 } else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) {
2483 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2484 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
2485 } else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") ||
2486 Name.startswith("avx512.cvtw2mask.") ||
2487 Name.startswith("avx512.cvtd2mask.") ||
2488 Name.startswith("avx512.cvtq2mask."))) {
2489 Value *Op = CI->getArgOperand(0);
2490 Value *Zero = llvm::Constant::getNullValue(Op->getType());
2491 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
2492 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr);
2493 } else if(IsX86 && (Name == "ssse3.pabs.b.128" ||
2494 Name == "ssse3.pabs.w.128" ||
2495 Name == "ssse3.pabs.d.128" ||
2496 Name.startswith("avx2.pabs") ||
2497 Name.startswith("avx512.mask.pabs"))) {
2498 Rep = upgradeAbs(Builder, *CI);
2499 } else if (IsX86 && (Name == "sse41.pmaxsb" ||
2500 Name == "sse2.pmaxs.w" ||
2501 Name == "sse41.pmaxsd" ||
2502 Name.startswith("avx2.pmaxs") ||
2503 Name.startswith("avx512.mask.pmaxs"))) {
2504 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
2505 } else if (IsX86 && (Name == "sse2.pmaxu.b" ||
2506 Name == "sse41.pmaxuw" ||
2507 Name == "sse41.pmaxud" ||
2508 Name.startswith("avx2.pmaxu") ||
2509 Name.startswith("avx512.mask.pmaxu"))) {
2510 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
2511 } else if (IsX86 && (Name == "sse41.pminsb" ||
2512 Name == "sse2.pmins.w" ||
2513 Name == "sse41.pminsd" ||
2514 Name.startswith("avx2.pmins") ||
2515 Name.startswith("avx512.mask.pmins"))) {
2516 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
2517 } else if (IsX86 && (Name == "sse2.pminu.b" ||
2518 Name == "sse41.pminuw" ||
2519 Name == "sse41.pminud" ||
2520 Name.startswith("avx2.pminu") ||
2521 Name.startswith("avx512.mask.pminu"))) {
2522 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
2523 } else if (IsX86 && (Name == "sse2.pmulu.dq" ||
2524 Name == "avx2.pmulu.dq" ||
2525 Name == "avx512.pmulu.dq.512" ||
2526 Name.startswith("avx512.mask.pmulu.dq."))) {
2527 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
2528 } else if (IsX86 && (Name == "sse41.pmuldq" ||
2529 Name == "avx2.pmul.dq" ||
2530 Name == "avx512.pmul.dq.512" ||
2531 Name.startswith("avx512.mask.pmul.dq."))) {
2532 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
2533 } else if (IsX86 && (Name == "sse.cvtsi2ss" ||
2534 Name == "sse2.cvtsi2sd" ||
2535 Name == "sse.cvtsi642ss" ||
2536 Name == "sse2.cvtsi642sd")) {
2537 Rep = Builder.CreateSIToFP(
2538 CI->getArgOperand(1),
2539 cast<VectorType>(CI->getType())->getElementType());
2540 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2541 } else if (IsX86 && Name == "avx512.cvtusi2sd") {
2542 Rep = Builder.CreateUIToFP(
2543 CI->getArgOperand(1),
2544 cast<VectorType>(CI->getType())->getElementType());
2545 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2546 } else if (IsX86 && Name == "sse2.cvtss2sd") {
2547 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
2548 Rep = Builder.CreateFPExt(
2549 Rep, cast<VectorType>(CI->getType())->getElementType());
2550 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2551 } else if (IsX86 && (Name == "sse2.cvtdq2pd" ||
2552 Name == "sse2.cvtdq2ps" ||
2553 Name == "avx.cvtdq2.pd.256" ||
2554 Name == "avx.cvtdq2.ps.256" ||
2555 Name.startswith("avx512.mask.cvtdq2pd.") ||
2556 Name.startswith("avx512.mask.cvtudq2pd.") ||
2557 Name.startswith("avx512.mask.cvtdq2ps.") ||
2558 Name.startswith("avx512.mask.cvtudq2ps.") ||
2559 Name.startswith("avx512.mask.cvtqq2pd.") ||
2560 Name.startswith("avx512.mask.cvtuqq2pd.") ||
2561 Name == "avx512.mask.cvtqq2ps.256" ||
2562 Name == "avx512.mask.cvtqq2ps.512" ||
2563 Name == "avx512.mask.cvtuqq2ps.256" ||
2564 Name == "avx512.mask.cvtuqq2ps.512" ||
2565 Name == "sse2.cvtps2pd" ||
2566 Name == "avx.cvt.ps2.pd.256" ||
2567 Name == "avx512.mask.cvtps2pd.128" ||
2568 Name == "avx512.mask.cvtps2pd.256")) {
2569 auto *DstTy = cast<FixedVectorType>(CI->getType());
2570 Rep = CI->getArgOperand(0);
2571 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2572
2573 unsigned NumDstElts = DstTy->getNumElements();
2574 if (NumDstElts < SrcTy->getNumElements()) {
2575 assert(NumDstElts == 2 && "Unexpected vector size")(static_cast <bool> (NumDstElts == 2 && "Unexpected vector size"
) ? void (0) : __assert_fail ("NumDstElts == 2 && \"Unexpected vector size\""
, "llvm/lib/IR/AutoUpgrade.cpp", 2575, __extension__ __PRETTY_FUNCTION__
));
2576 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
2577 }
2578
2579 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2580 bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));
2581 if (IsPS2PD)
2582 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
2583 else if (CI->arg_size() == 4 &&
2584 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2585 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2586 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2587 : Intrinsic::x86_avx512_sitofp_round;
2588 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
2589 { DstTy, SrcTy });
2590 Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
2591 } else {
2592 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
2593 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
2594 }
2595
2596 if (CI->arg_size() >= 3)
2597 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2598 CI->getArgOperand(1));
2599 } else if (IsX86 && (Name.startswith("avx512.mask.vcvtph2ps.") ||
2600 Name.startswith("vcvtph2ps."))) {
2601 auto *DstTy = cast<FixedVectorType>(CI->getType());
2602 Rep = CI->getArgOperand(0);
2603 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2604 unsigned NumDstElts = DstTy->getNumElements();
2605 if (NumDstElts != SrcTy->getNumElements()) {
2606 assert(NumDstElts == 4 && "Unexpected vector size")(static_cast <bool> (NumDstElts == 4 && "Unexpected vector size"
) ? void (0) : __assert_fail ("NumDstElts == 4 && \"Unexpected vector size\""
, "llvm/lib/IR/AutoUpgrade.cpp", 2606, __extension__ __PRETTY_FUNCTION__
));
2607 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
2608 }
2609 Rep = Builder.CreateBitCast(
2610 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
2611 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
2612 if (CI->arg_size() >= 3)
2613 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2614 CI->getArgOperand(1));
2615 } else if (IsX86 && Name.startswith("avx512.mask.load")) {
2616 // "avx512.mask.loadu." or "avx512.mask.load."
2617 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
2618 Rep =
2619 UpgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2620 CI->getArgOperand(2), Aligned);
2621 } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {
2622 auto *ResultTy = cast<FixedVectorType>(CI->getType());
2623 Type *PtrTy = ResultTy->getElementType();
2624
2625 // Cast the pointer to element type.
2626 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2627 llvm::PointerType::getUnqual(PtrTy));
2628
2629 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2630 ResultTy->getNumElements());
2631
2632 Function *ELd = Intrinsic::getDeclaration(F->getParent(),
2633 Intrinsic::masked_expandload,
2634 ResultTy);
2635 Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
2636 } else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) {
2637 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
2638 Type *PtrTy = ResultTy->getElementType();
2639
2640 // Cast the pointer to element type.
2641 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2642 llvm::PointerType::getUnqual(PtrTy));
2643
2644 Value *MaskVec =
2645 getX86MaskVec(Builder, CI->getArgOperand(2),
2646 cast<FixedVectorType>(ResultTy)->getNumElements());
2647
2648 Function *CSt = Intrinsic::getDeclaration(F->getParent(),
2649 Intrinsic::masked_compressstore,
2650 ResultTy);
2651 Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
2652 } else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
2653 Name.startswith("avx512.mask.expand."))) {
2654 auto *ResultTy = cast<FixedVectorType>(CI->getType());
2655
2656 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2657 ResultTy->getNumElements());
2658
2659 bool IsCompress = Name[12] == 'c';
2660 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2661 : Intrinsic::x86_avx512_mask_expand;
2662 Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
2663 Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
2664 MaskVec });
2665 } else if (IsX86 && Name.startswith("xop.vpcom")) {
2666 bool IsSigned;
2667 if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||
2668 Name.endswith("uq"))
2669 IsSigned = false;
2670 else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||
2671 Name.endswith("q"))
2672 IsSigned = true;
2673 else
2674 llvm_unreachable("Unknown suffix")::llvm::llvm_unreachable_internal("Unknown suffix", "llvm/lib/IR/AutoUpgrade.cpp"
, 2674);
2675
2676 unsigned Imm;
2677 if (CI->arg_size() == 3) {
2678 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2679 } else {
2680 Name = Name.substr(9); // strip off "xop.vpcom"
2681 if (Name.startswith("lt"))
2682 Imm = 0;
2683 else if (Name.startswith("le"))
2684 Imm = 1;
2685 else if (Name.startswith("gt"))
2686 Imm = 2;
2687 else if (Name.startswith("ge"))
2688 Imm = 3;
2689 else if (Name.startswith("eq"))
2690 Imm = 4;
2691 else if (Name.startswith("ne"))
2692 Imm = 5;
2693 else if (Name.startswith("false"))
2694 Imm = 6;
2695 else if (Name.startswith("true"))
2696 Imm = 7;
2697 else
2698 llvm_unreachable("Unknown condition")::llvm::llvm_unreachable_internal("Unknown condition", "llvm/lib/IR/AutoUpgrade.cpp"
, 2698);
2699 }
2700
2701 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
2702 } else if (IsX86 && Name.startswith("xop.vpcmov")) {
2703 Value *Sel = CI->getArgOperand(2);
2704 Value *NotSel = Builder.CreateNot(Sel);
2705 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
2706 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
2707 Rep = Builder.CreateOr(Sel0, Sel1);
2708 } else if (IsX86 && (Name.startswith("xop.vprot") ||
2709 Name.startswith("avx512.prol") ||
2710 Name.startswith("avx512.mask.prol"))) {
2711 Rep = upgradeX86Rotate(Builder, *CI, false);
2712 } else if (IsX86 && (Name.startswith("avx512.pror") ||
2713 Name.startswith("avx512.mask.pror"))) {
2714 Rep = upgradeX86Rotate(Builder, *CI, true);
2715 } else if (IsX86 && (Name.startswith("avx512.vpshld.") ||
2716 Name.startswith("avx512.mask.vpshld") ||
2717 Name.startswith("avx512.maskz.vpshld"))) {
2718 bool ZeroMask = Name[11] == 'z';
2719 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
2720 } else if (IsX86 && (Name.startswith("avx512.vpshrd.") ||
2721 Name.startswith("avx512.mask.vpshrd") ||
2722 Name.startswith("avx512.maskz.vpshrd"))) {
2723 bool ZeroMask = Name[11] == 'z';
2724 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
2725 } else if (IsX86 && Name == "sse42.crc32.64.8") {
2726 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
2727 Intrinsic::x86_sse42_crc32_32_8);
2728 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
2729 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
2730 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
2731 } else if (IsX86 && (Name.startswith("avx.vbroadcast.s") ||
2732 Name.startswith("avx512.vbroadcast.s"))) {
2733 // Replace broadcasts with a series of insertelements.
2734 auto *VecTy = cast<FixedVectorType>(CI->getType());
2735 Type *EltTy = VecTy->getElementType();
2736 unsigned EltNum = VecTy->getNumElements();
2737 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
2738 EltTy->getPointerTo());
2739 Value *Load = Builder.CreateLoad(EltTy, Cast);
2740 Type *I32Ty = Type::getInt32Ty(C);
2741 Rep = PoisonValue::get(VecTy);
2742 for (unsigned I = 0; I < EltNum; ++I)
2743 Rep = Builder.CreateInsertElement(Rep, Load,
2744 ConstantInt::get(I32Ty, I));
2745 } else if (IsX86 && (Name.startswith("sse41.pmovsx") ||
2746 Name.startswith("sse41.pmovzx") ||
2747 Name.startswith("avx2.pmovsx") ||
2748 Name.startswith("avx2.pmovzx") ||
2749 Name.startswith("avx512.mask.pmovsx") ||
2750 Name.startswith("avx512.mask.pmovzx"))) {
2751 auto *DstTy = cast<FixedVectorType>(CI->getType());
2752 unsigned NumDstElts = DstTy->getNumElements();
2753
2754 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
2755 SmallVector<int, 8> ShuffleMask(NumDstElts);
2756 for (unsigned i = 0; i != NumDstElts; ++i)
2757 ShuffleMask[i] = i;
2758
2759 Value *SV =
2760 Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
2761
2762 bool DoSext = (StringRef::npos != Name.find("pmovsx"));
2763 Rep = DoSext ? Builder.CreateSExt(SV, DstTy)
2764 : Builder.CreateZExt(SV, DstTy);
2765 // If there are 3 arguments, it's a masked intrinsic so we need a select.
2766 if (CI->arg_size() == 3)
2767 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2768 CI->getArgOperand(1));
2769 } else if (Name == "avx512.mask.pmov.qd.256" ||
2770 Name == "avx512.mask.pmov.qd.512" ||
2771 Name == "avx512.mask.pmov.wb.256" ||
2772 Name == "avx512.mask.pmov.wb.512") {
2773 Type *Ty = CI->getArgOperand(1)->getType();
2774 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
2775 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2776 CI->getArgOperand(1));
2777 } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
2778 Name == "avx2.vbroadcasti128")) {
2779 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
2780 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
2781 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
2782 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
2783 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
2784 PointerType::getUnqual(VT));
2785 Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
2786 if (NumSrcElts == 2)
2787 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
2788 else
2789 Rep = Builder.CreateShuffleVector(
2790 Load, ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
2791 } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
2792 Name.startswith("avx512.mask.shuf.f"))) {
2793 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2794 Type *VT = CI->getType();
2795 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
2796 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
2797 unsigned ControlBitsMask = NumLanes - 1;
2798 unsigned NumControlBits = NumLanes / 2;
2799 SmallVector<int, 8> ShuffleMask(0);
2800
2801 for (unsigned l = 0; l != NumLanes; ++l) {
2802 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
2803 // We actually need the other source.
2804 if (l >= NumLanes / 2)
2805 LaneMask += NumLanes;
2806 for (unsigned i = 0; i != NumElementsInLane; ++i)
2807 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
2808 }
2809 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2810 CI->getArgOperand(1), ShuffleMask);
2811 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2812 CI->getArgOperand(3));
2813 }else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") ||
2814 Name.startswith("avx512.mask.broadcasti"))) {
2815 unsigned NumSrcElts =
2816 cast<FixedVectorType>(CI->getArgOperand(0)->getType())
2817 ->getNumElements();
2818 unsigned NumDstElts =
2819 cast<FixedVectorType>(CI->getType())->getNumElements();
2820
2821 SmallVector<int, 8> ShuffleMask(NumDstElts);
2822 for (unsigned i = 0; i != NumDstElts; ++i)
2823 ShuffleMask[i] = i % NumSrcElts;
2824
2825 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2826 CI->getArgOperand(0),
2827 ShuffleMask);
2828 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2829 CI->getArgOperand(1));
2830 } else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||
2831 Name.startswith("avx2.vbroadcast") ||
2832 Name.startswith("avx512.pbroadcast") ||
2833 Name.startswith("avx512.mask.broadcast.s"))) {
2834 // Replace vp?broadcasts with a vector shuffle.
2835 Value *Op = CI->getArgOperand(0);
2836 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
2837 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
2838 SmallVector<int, 8> M;
2839 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
2840 Rep = Builder.CreateShuffleVector(Op, M);
2841
2842 if (CI->arg_size() == 3)
2843 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2844 CI->getArgOperand(1));
2845 } else if (IsX86 && (Name.startswith("sse2.padds.") ||
2846 Name.startswith("avx2.padds.") ||
2847 Name.startswith("avx512.padds.") ||
2848 Name.startswith("avx512.mask.padds."))) {
2849 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
2850 } else if (IsX86 && (Name.startswith("sse2.psubs.") ||
2851 Name.startswith("avx2.psubs.") ||
2852 Name.startswith("avx512.psubs.") ||
2853 Name.startswith("avx512.mask.psubs."))) {
2854 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
2855 } else if (IsX86 && (Name.startswith("sse2.paddus.") ||
2856 Name.startswith("avx2.paddus.") ||
2857 Name.startswith("avx512.mask.paddus."))) {
2858 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
2859 } else if (IsX86 && (Name.startswith("sse2.psubus.") ||
2860 Name.startswith("avx2.psubus.") ||
2861 Name.startswith("avx512.mask.psubus."))) {
2862 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
2863 } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
2864 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2865 CI->getArgOperand(1),
2866 CI->getArgOperand(2),
2867 CI->getArgOperand(3),
2868 CI->getArgOperand(4),
2869 false);
2870 } else if (IsX86 && Name.startswith("avx512.mask.valign.")) {
2871 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2872 CI->getArgOperand(1),
2873 CI->getArgOperand(2),
2874 CI->getArgOperand(3),
2875 CI->getArgOperand(4),
2876 true);
2877 } else if (IsX86 && (Name == "sse2.psll.dq" ||
2878 Name == "avx2.psll.dq")) {
2879 // 128/256-bit shift left specified in bits.
2880 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2881 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
2882 Shift / 8); // Shift is in bits.
2883 } else if (IsX86 && (Name == "sse2.psrl.dq" ||
2884 Name == "avx2.psrl.dq")) {
2885 // 128/256-bit shift right specified in bits.
2886 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2887 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
2888 Shift / 8); // Shift is in bits.
2889 } else if (IsX86 && (Name == "sse2.psll.dq.bs" ||
2890 Name == "avx2.psll.dq.bs" ||
2891 Name == "avx512.psll.dq.512")) {
2892 // 128/256/512-bit shift left specified in bytes.
2893 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2894 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2895 } else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||
2896 Name == "avx2.psrl.dq.bs" ||
2897 Name == "avx512.psrl.dq.512")) {
2898 // 128/256/512-bit shift right specified in bytes.
2899 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2900 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2901 } else if (IsX86 && (Name == "sse41.pblendw" ||
2902 Name.startswith("sse41.blendp") ||
2903 Name.startswith("avx.blend.p") ||
2904 Name == "avx2.pblendw" ||
2905 Name.startswith("avx2.pblendd."))) {
2906 Value *Op0 = CI->getArgOperand(0);
2907 Value *Op1 = CI->getArgOperand(1);
2908 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2909 auto *VecTy = cast<FixedVectorType>(CI->getType());
2910 unsigned NumElts = VecTy->getNumElements();
2911
2912 SmallVector<int, 16> Idxs(NumElts);
2913 for (unsigned i = 0; i != NumElts; ++i)
2914 Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
2915
2916 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2917 } else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||
2918 Name == "avx2.vinserti128" ||
2919 Name.startswith("avx512.mask.insert"))) {
2920 Value *Op0 = CI->getArgOperand(0);
2921 Value *Op1 = CI->getArgOperand(1);
2922 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2923 unsigned DstNumElts =
2924 cast<FixedVectorType>(CI->getType())->getNumElements();
2925 unsigned SrcNumElts =
2926 cast<FixedVectorType>(Op1->getType())->getNumElements();
2927 unsigned Scale = DstNumElts / SrcNumElts;
2928
2929 // Mask off the high bits of the immediate value; hardware ignores those.
2930 Imm = Imm % Scale;
2931
2932 // Extend the second operand into a vector the size of the destination.
2933 SmallVector<int, 8> Idxs(DstNumElts);
2934 for (unsigned i = 0; i != SrcNumElts; ++i)
2935 Idxs[i] = i;
2936 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
2937 Idxs[i] = SrcNumElts;
2938 Rep = Builder.CreateShuffleVector(Op1, Idxs);
2939
2940 // Insert the second operand into the first operand.
2941
2942 // Note that there is no guarantee that instruction lowering will actually
2943 // produce a vinsertf128 instruction for the created shuffles. In
2944 // particular, the 0 immediate case involves no lane changes, so it can
2945 // be handled as a blend.
2946
2947 // Example of shuffle mask for 32-bit elements:
2948 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
2949 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
2950
2951 // First fill with identify mask.
2952 for (unsigned i = 0; i != DstNumElts; ++i)
2953 Idxs[i] = i;
2954 // Then replace the elements where we need to insert.
2955 for (unsigned i = 0; i != SrcNumElts; ++i)
2956 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
2957 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
2958
2959 // If the intrinsic has a mask operand, handle that.
2960 if (CI->arg_size() == 5)
2961 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2962 CI->getArgOperand(3));
2963 } else if (IsX86 && (Name.startswith("avx.vextractf128.") ||
2964 Name == "avx2.vextracti128" ||
2965 Name.startswith("avx512.mask.vextract"))) {
2966 Value *Op0 = CI->getArgOperand(0);
2967 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2968 unsigned DstNumElts =
2969 cast<FixedVectorType>(CI->getType())->getNumElements();
2970 unsigned SrcNumElts =
2971 cast<FixedVectorType>(Op0->getType())->getNumElements();
2972 unsigned Scale = SrcNumElts / DstNumElts;
2973
2974 // Mask off the high bits of the immediate value; hardware ignores those.
2975 Imm = Imm % Scale;
2976
2977 // Get indexes for the subvector of the input vector.
2978 SmallVector<int, 8> Idxs(DstNumElts);
2979 for (unsigned i = 0; i != DstNumElts; ++i) {
2980 Idxs[i] = i + (Imm * DstNumElts);
2981 }
2982 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2983
2984 // If the intrinsic has a mask operand, handle that.
2985 if (CI->arg_size() == 4)
2986 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2987 CI->getArgOperand(2));
2988 } else if (!IsX86 && Name == "stackprotectorcheck") {
2989 Rep = nullptr;
2990 } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||
2991 Name.startswith("avx512.mask.perm.di."))) {
2992 Value *Op0 = CI->getArgOperand(0);
2993 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2994 auto *VecTy = cast<FixedVectorType>(CI->getType());
2995 unsigned NumElts = VecTy->getNumElements();
2996
2997 SmallVector<int, 8> Idxs(NumElts);
2998 for (unsigned i = 0; i != NumElts; ++i)
2999 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3000
3001 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3002
3003 if (CI->arg_size() == 4)
3004 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3005 CI->getArgOperand(2));
3006 } else if (IsX86 && (Name.startswith("avx.vperm2f128.") ||
3007 Name == "avx2.vperm2i128")) {
3008 // The immediate permute control byte looks like this:
3009 // [1:0] - select 128 bits from sources for low half of destination
3010 // [2] - ignore
3011 // [3] - zero low half of destination
3012 // [5:4] - select 128 bits from sources for high half of destination
3013 // [6] - ignore
3014 // [7] - zero high half of destination
3015
3016 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3017
3018 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3019 unsigned HalfSize = NumElts / 2;
3020 SmallVector<int, 8> ShuffleMask(NumElts);
3021
3022 // Determine which operand(s) are actually in use for this instruction.
3023 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3024 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3025
3026 // If needed, replace operands based on zero mask.
3027 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
3028 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
3029
3030 // Permute low half of result.
3031 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3032 for (unsigned i = 0; i < HalfSize; ++i)
3033 ShuffleMask[i] = StartIndex + i;
3034
3035 // Permute high half of result.
3036 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3037 for (unsigned i = 0; i < HalfSize; ++i)
3038 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3039
3040 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3041
3042 } else if (IsX86 && (Name.startswith("avx.vpermil.") ||
3043 Name == "sse2.pshuf.d" ||
3044 Name.startswith("avx512.mask.vpermil.p") ||
3045 Name.startswith("avx512.mask.pshuf.d."))) {
3046 Value *Op0 = CI->getArgOperand(0);
3047 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3048 auto *VecTy = cast<FixedVectorType>(CI->getType());
3049 unsigned NumElts = VecTy->getNumElements();
3050 // Calculate the size of each index in the immediate.
3051 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3052 unsigned IdxMask = ((1 << IdxSize) - 1);
3053
3054 SmallVector<int, 8> Idxs(NumElts);
3055 // Lookup the bits for this element, wrapping around the immediate every
3056 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
3057 // to offset by the first index of each group.
3058 for (unsigned i = 0; i != NumElts; ++i)
3059 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3060
3061 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3062
3063 if (CI->arg_size() == 4)
3064 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3065 CI->getArgOperand(2));
3066 } else if (IsX86 && (Name == "sse2.pshufl.w" ||
3067 Name.startswith("avx512.mask.pshufl.w."))) {
3068 Value *Op0 = CI->getArgOperand(0);
3069 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3070 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3071
3072 SmallVector<int, 16> Idxs(NumElts);
3073 for (unsigned l = 0; l != NumElts; l += 8) {
3074 for (unsigned i = 0; i != 4; ++i)
3075 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3076 for (unsigned i = 4; i != 8; ++i)
3077 Idxs[i + l] = i + l;
3078 }
3079
3080 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3081
3082 if (CI->arg_size() == 4)
3083 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3084 CI->getArgOperand(2));
3085 } else if (IsX86 && (Name == "sse2.pshufh.w" ||
3086 Name.startswith("avx512.mask.pshufh.w."))) {
3087 Value *Op0 = CI->getArgOperand(0);
3088 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3089 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3090
3091 SmallVector<int, 16> Idxs(NumElts);
3092 for (unsigned l = 0; l != NumElts; l += 8) {
3093 for (unsigned i = 0; i != 4; ++i)
3094 Idxs[i + l] = i + l;
3095 for (unsigned i = 0; i != 4; ++i)
3096 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3097 }
3098
3099 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3100
3101 if (CI->arg_size() == 4)
3102 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3103 CI->getArgOperand(2));
3104 } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {
3105 Value *Op0 = CI->getArgOperand(0);
3106 Value *Op1 = CI->getArgOperand(1);
3107 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3108 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3109
3110 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
3111 unsigned HalfLaneElts = NumLaneElts / 2;
3112
3113 SmallVector<int, 16> Idxs(NumElts);
3114 for (unsigned i = 0; i != NumElts; ++i) {
3115 // Base index is the starting element of the lane.
3116 Idxs[i] = i - (i % NumLaneElts);
3117 // If we are half way through the lane switch to the other source.
3118 if ((i % NumLaneElts) >= HalfLaneElts)
3119 Idxs[i] += NumElts;
3120 // Now select the specific element. By adding HalfLaneElts bits from
3121 // the immediate. Wrapping around the immediate every 8-bits.
3122 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3123 }
3124
3125 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3126
3127 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
3128 CI->getArgOperand(3));
3129 } else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||
3130 Name.startswith("avx512.mask.movshdup") ||
3131 Name.startswith("avx512.mask.movsldup"))) {
3132 Value *Op0 = CI->getArgOperand(0);
3133 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3134 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
3135
3136 unsigned Offset = 0;
3137 if (Name.startswith("avx512.mask.movshdup."))
3138 Offset = 1;
3139
3140 SmallVector<int, 16> Idxs(NumElts);
3141 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
3142 for (unsigned i = 0; i != NumLaneElts; i += 2) {
3143 Idxs[i + l + 0] = i + l + Offset;
3144 Idxs[i + l + 1] = i + l + Offset;
3145 }
3146
3147 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3148
3149 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
3150 CI->getArgOperand(1));
3151 } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||
3152 Name.startswith("avx512.mask.unpckl."))) {
3153 Value *Op0 = CI->getArgOperand(0);
3154 Value *Op1 = CI->getArgOperand(1);
3155 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3156 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
3157
3158 SmallVector<int, 64> Idxs(NumElts);
3159 for (int l = 0; l != NumElts; l += NumLaneElts)
3160 for (int i = 0; i != NumLaneElts; ++i)
3161 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3162
3163 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3164
3165 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3166 CI->getArgOperand(2));
3167 } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||
3168 Name.startswith("avx512.mask.unpckh."))) {
3169 Value *Op0 = CI->getArgOperand(0);
3170 Value *Op1 = CI->getArgOperand(1);
3171 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3172 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
3173
3174 SmallVector<int, 64> Idxs(NumElts);
3175 for (int l = 0; l != NumElts; l += NumLaneElts)
3176 for (int i = 0; i != NumLaneElts; ++i)
3177 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3178
3179 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3180
3181 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3182 CI->getArgOperand(2));
3183 } else if (IsX86 && (Name.startswith("avx512.mask.and.") ||
3184 Name.startswith("avx512.mask.pand."))) {
3185 VectorType *FTy = cast<VectorType>(CI->getType());
3186 VectorType *ITy = VectorType::getInteger(FTy);
3187 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3188 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3189 Rep = Builder.CreateBitCast(Rep, FTy);
3190 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3191 CI->getArgOperand(2));
3192 } else if (IsX86 && (Name.startswith("avx512.mask.andn.") ||
3193 Name.startswith("avx512.mask.pandn."))) {
3194 VectorType *FTy = cast<VectorType>(CI->getType());
3195 VectorType *ITy = VectorType::getInteger(FTy);
3196 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
3197 Rep = Builder.CreateAnd(Rep,
3198 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3199 Rep = Builder.CreateBitCast(Rep, FTy);
3200 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3201 CI->getArgOperand(2));
3202 } else if (IsX86 && (Name.startswith("avx512.mask.or.") ||
3203 Name.startswith("avx512.mask.por."))) {
3204 VectorType *FTy = cast<VectorType>(CI->getType());
3205 VectorType *ITy = VectorType::getInteger(FTy);
3206 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3207 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3208 Rep = Builder.CreateBitCast(Rep, FTy);
3209 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3210 CI->getArgOperand(2));
3211 } else if (IsX86 && (Name.startswith("avx512.mask.xor.") ||
3212 Name.startswith("avx512.mask.pxor."))) {
3213 VectorType *FTy = cast<VectorType>(CI->getType());
3214 VectorType *ITy = VectorType::getInteger(FTy);
3215 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3216 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3217 Rep = Builder.CreateBitCast(Rep, FTy);
3218 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3219 CI->getArgOperand(2));
3220 } else if (IsX86 && Name.startswith("avx512.mask.padd.")) {
3221 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3222 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3223 CI->getArgOperand(2));
3224 } else if (IsX86 && Name.startswith("avx512.mask.psub.")) {
3225 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
3226 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3227 CI->getArgOperand(2));
3228 } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {
3229 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
3230 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3231 CI->getArgOperand(2));
3232 } else if (IsX86 && Name.startswith("avx512.mask.add.p")) {
3233 if (Name.endswith(".512")) {
3234 Intrinsic::ID IID;
3235 if (Name[17] == 's')
3236 IID = Intrinsic::x86_avx512_add_ps_512;
3237 else
3238 IID = Intrinsic::x86_avx512_add_pd_512;
3239
3240 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3241 { CI->getArgOperand(0), CI->getArgOperand(1),
3242 CI->getArgOperand(4) });
3243 } else {
3244 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3245 }
3246 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3247 CI->getArgOperand(2));
3248 } else if (IsX86 && Name.startswith("avx512.mask.div.p")) {
3249 if (Name.endswith(".512")) {
3250 Intrinsic::ID IID;
3251 if (Name[17] == 's')
3252 IID = Intrinsic::x86_avx512_div_ps_512;
3253 else
3254 IID = Intrinsic::x86_avx512_div_pd_512;
3255
3256 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3257 { CI->getArgOperand(0), CI->getArgOperand(1),
3258 CI->getArgOperand(4) });
3259 } else {
3260 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
3261 }
3262 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3263 CI->getArgOperand(2));
3264 } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {
3265 if (Name.endswith(".512")) {
3266 Intrinsic::ID IID;
3267 if (Name[17] == 's')
3268 IID = Intrinsic::x86_avx512_mul_ps_512;
3269 else
3270 IID = Intrinsic::x86_avx512_mul_pd_512;
3271
3272 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3273 { CI->getArgOperand(0), CI->getArgOperand(1),
3274 CI->getArgOperand(4) });
3275 } else {
3276 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
3277 }
3278 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3279 CI->getArgOperand(2));
3280 } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {
3281 if (Name.endswith(".512")) {
3282 Intrinsic::ID IID;
3283 if (Name[17] == 's')
3284 IID = Intrinsic::x86_avx512_sub_ps_512;
3285 else
3286 IID = Intrinsic::x86_avx512_sub_pd_512;
3287
3288 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3289 { CI->getArgOperand(0), CI->getArgOperand(1),
3290 CI->getArgOperand(4) });
3291 } else {
3292 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
3293 }
3294 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3295 CI->getArgOperand(2));
3296 } else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
3297 Name.startswith("avx512.mask.min.p")) &&
3298 Name.drop_front(18) == ".512") {
3299 bool IsDouble = Name[17] == 'd';
3300 bool IsMin = Name[13] == 'i';
3301 static const Intrinsic::ID MinMaxTbl[2][2] = {
3302 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
3303 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
3304 };
3305 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
3306
3307 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3308 { CI->getArgOperand(0), CI->getArgOperand(1),
3309 CI->getArgOperand(4) });
3310 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3311 CI->getArgOperand(2));
3312 } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) {
3313 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
3314 Intrinsic::ctlz,
3315 CI->getType()),
3316 { CI->getArgOperand(0), Builder.getInt1(false) });
3317 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
3318 CI->getArgOperand(1));
3319 } else if (IsX86 && Name.startswith("avx512.mask.psll")) {
3320 bool IsImmediate = Name[16] == 'i' ||
3321 (Name.size() > 18 && Name[18] == 'i');
3322 bool IsVariable = Name[16] == 'v';
3323 char Size = Name[16] == '.' ? Name[17] :
3324 Name[17] == '.' ? Name[18] :
3325 Name[18] == '.' ? Name[19] :
3326 Name[20];
3327
3328 Intrinsic::ID IID;
3329 if (IsVariable && Name[17] != '.') {
3330 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
3331 IID = Intrinsic::x86_avx2_psllv_q;
3332 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
3333 IID = Intrinsic::x86_avx2_psllv_q_256;
3334 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
3335 IID = Intrinsic::x86_avx2_psllv_d;
3336 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
3337 IID = Intrinsic::x86_avx2_psllv_d_256;
3338 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
3339 IID = Intrinsic::x86_avx512_psllv_w_128;
3340 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
3341 IID = Intrinsic::x86_avx512_psllv_w_256;
3342 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
3343 IID = Intrinsic::x86_avx512_psllv_w_512;
3344 else
3345 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3345);
3346 } else if (Name.endswith(".128")) {
3347 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
3348 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3349 : Intrinsic::x86_sse2_psll_d;
3350 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
3351 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3352 : Intrinsic::x86_sse2_psll_q;
3353 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
3354 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3355 : Intrinsic::x86_sse2_psll_w;
3356 else
3357 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3357);
3358 } else if (Name.endswith(".256")) {
3359 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
3360 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3361 : Intrinsic::x86_avx2_psll_d;
3362 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
3363 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3364 : Intrinsic::x86_avx2_psll_q;
3365 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
3366 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3367 : Intrinsic::x86_avx2_psll_w;
3368 else
3369 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3369);
3370 } else {
3371 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
3372 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3373 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
3374 Intrinsic::x86_avx512_psll_d_512;
3375 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
3376 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3377 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
3378 Intrinsic::x86_avx512_psll_q_512;
3379 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
3380 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3381 : Intrinsic::x86_avx512_psll_w_512;
3382 else
3383 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3383);
3384 }
3385
3386 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3387 } else if (IsX86 && Name.startswith("avx512.mask.psrl")) {
3388 bool IsImmediate = Name[16] == 'i' ||
3389 (Name.size() > 18 && Name[18] == 'i');
3390 bool IsVariable = Name[16] == 'v';
3391 char Size = Name[16] == '.' ? Name[17] :
3392 Name[17] == '.' ? Name[18] :
3393 Name[18] == '.' ? Name[19] :
3394 Name[20];
3395
3396 Intrinsic::ID IID;
3397 if (IsVariable && Name[17] != '.') {
3398 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
3399 IID = Intrinsic::x86_avx2_psrlv_q;
3400 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
3401 IID = Intrinsic::x86_avx2_psrlv_q_256;
3402 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
3403 IID = Intrinsic::x86_avx2_psrlv_d;
3404 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
3405 IID = Intrinsic::x86_avx2_psrlv_d_256;
3406 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
3407 IID = Intrinsic::x86_avx512_psrlv_w_128;
3408 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
3409 IID = Intrinsic::x86_avx512_psrlv_w_256;
3410 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
3411 IID = Intrinsic::x86_avx512_psrlv_w_512;
3412 else
3413 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3413);
3414 } else if (Name.endswith(".128")) {
3415 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
3416 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3417 : Intrinsic::x86_sse2_psrl_d;
3418 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
3419 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3420 : Intrinsic::x86_sse2_psrl_q;
3421 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
3422 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3423 : Intrinsic::x86_sse2_psrl_w;
3424 else
3425 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3425);
3426 } else if (Name.endswith(".256")) {
3427 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
3428 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3429 : Intrinsic::x86_avx2_psrl_d;
3430 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
3431 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3432 : Intrinsic::x86_avx2_psrl_q;
3433 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
3434 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3435 : Intrinsic::x86_avx2_psrl_w;
3436 else
3437 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3437);
3438 } else {
3439 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
3440 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3441 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
3442 Intrinsic::x86_avx512_psrl_d_512;
3443 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
3444 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3445 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
3446 Intrinsic::x86_avx512_psrl_q_512;
3447 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
3448 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3449 : Intrinsic::x86_avx512_psrl_w_512;
3450 else
3451 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3451);
3452 }
3453
3454 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3455 } else if (IsX86 && Name.startswith("avx512.mask.psra")) {
3456 bool IsImmediate = Name[16] == 'i' ||
3457 (Name.size() > 18 && Name[18] == 'i');
3458 bool IsVariable = Name[16] == 'v';
3459 char Size = Name[16] == '.' ? Name[17] :
3460 Name[17] == '.' ? Name[18] :
3461 Name[18] == '.' ? Name[19] :
3462 Name[20];
3463
3464 Intrinsic::ID IID;
3465 if (IsVariable && Name[17] != '.') {
3466 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
3467 IID = Intrinsic::x86_avx2_psrav_d;
3468 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
3469 IID = Intrinsic::x86_avx2_psrav_d_256;
3470 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
3471 IID = Intrinsic::x86_avx512_psrav_w_128;
3472 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
3473 IID = Intrinsic::x86_avx512_psrav_w_256;
3474 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
3475 IID = Intrinsic::x86_avx512_psrav_w_512;
3476 else
3477 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3477);
3478 } else if (Name.endswith(".128")) {
3479 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
3480 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3481 : Intrinsic::x86_sse2_psra_d;
3482 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
3483 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3484 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
3485 Intrinsic::x86_avx512_psra_q_128;
3486 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
3487 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3488 : Intrinsic::x86_sse2_psra_w;
3489 else
3490 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3490);
3491 } else if (Name.endswith(".256")) {
3492 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
3493 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3494 : Intrinsic::x86_avx2_psra_d;
3495 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
3496 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3497 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
3498 Intrinsic::x86_avx512_psra_q_256;
3499 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
3500 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3501 : Intrinsic::x86_avx2_psra_w;
3502 else
3503 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3503);
3504 } else {
3505 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
3506 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3507 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
3508 Intrinsic::x86_avx512_psra_d_512;
3509 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
3510 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3511 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
3512 Intrinsic::x86_avx512_psra_q_512;
3513 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
3514 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3515 : Intrinsic::x86_avx512_psra_w_512;
3516 else
3517 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "llvm/lib/IR/AutoUpgrade.cpp"
, 3517);
3518 }
3519
3520 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3521 } else if (IsX86 && Name.startswith("avx512.mask.move.s")) {
3522 Rep = upgradeMaskedMove(Builder, *CI);
3523 } else if (IsX86 && Name.startswith("avx512.cvtmask2")) {
3524 Rep = UpgradeMaskToInt(Builder, *CI);
3525 } else if (IsX86 && Name.endswith(".movntdqa")) {
3526 Module *M = F->getParent();
3527 MDNode *Node = MDNode::get(
3528 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3529
3530 Value *Ptr = CI->getArgOperand(0);
3531
3532 // Convert the type of the pointer to a pointer to the stored type.
3533 Value *BC = Builder.CreateBitCast(
3534 Ptr, PointerType::getUnqual(CI->getType()), "cast");
3535 LoadInst *LI = Builder.CreateAlignedLoad(
3536 CI->getType(), BC,
3537 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3538 LI->setMetadata(M->getMDKindID("nontemporal"), Node);
3539 Rep = LI;
3540 } else if (IsX86 && (Name.startswith("fma.vfmadd.") ||
3541 Name.startswith("fma.vfmsub.") ||
3542 Name.startswith("fma.vfnmadd.") ||
3543 Name.startswith("fma.vfnmsub."))) {
3544 bool NegMul = Name[6] == 'n';
3545 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
3546 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
3547
3548 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3549 CI->getArgOperand(2) };
3550
3551 if (IsScalar) {
3552 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3553 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3554 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3555 }
3556
3557 if (NegMul && !IsScalar)
3558 Ops[0] = Builder.CreateFNeg(Ops[0]);
3559 if (NegMul && IsScalar)
3560 Ops[1] = Builder.CreateFNeg(Ops[1]);
3561 if (NegAcc)
3562 Ops[2] = Builder.CreateFNeg(Ops[2]);
3563
3564 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3565 Intrinsic::fma,
3566 Ops[0]->getType()),
3567 Ops);
3568
3569 if (IsScalar)
3570 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
3571 (uint64_t)0);
3572 } else if (IsX86 && Name.startswith("fma4.vfmadd.s")) {
3573 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3574 CI->getArgOperand(2) };
3575
3576 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3577 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3578 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3579
3580 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3581 Intrinsic::fma,
3582 Ops[0]->getType()),
3583 Ops);
3584
3585 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
3586 Rep, (uint64_t)0);
3587 } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.s") ||
3588 Name.startswith("avx512.maskz.vfmadd.s") ||
3589 Name.startswith("avx512.mask3.vfmadd.s") ||
3590 Name.startswith("avx512.mask3.vfmsub.s") ||
3591 Name.startswith("avx512.mask3.vfnmsub.s"))) {
3592 bool IsMask3 = Name[11] == '3';
3593 bool IsMaskZ = Name[11] == 'z';
3594 // Drop the "avx512.mask." to make it easier.
3595 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3596 bool NegMul = Name[2] == 'n';
3597 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3598
3599 Value *A = CI->getArgOperand(0);
3600 Value *B = CI->getArgOperand(1);
3601 Value *C = CI->getArgOperand(2);
3602
3603 if (NegMul && (IsMask3 || IsMaskZ))
3604 A = Builder.CreateFNeg(A);
3605 if (NegMul && !(IsMask3 || IsMaskZ))
3606 B = Builder.CreateFNeg(B);
3607 if (NegAcc)
3608 C = Builder.CreateFNeg(C);
3609
3610 A = Builder.CreateExtractElement(A, (uint64_t)0);
3611 B = Builder.CreateExtractElement(B, (uint64_t)0);
3612 C = Builder.CreateExtractElement(C, (uint64_t)0);
3613
3614 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3615 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
3616 Value *Ops[] = { A, B, C, CI->getArgOperand(4) };
3617
3618 Intrinsic::ID IID;
3619 if (Name.back() == 'd')
3620 IID = Intrinsic::x86_avx512_vfmadd_f64;
3621 else
3622 IID = Intrinsic::x86_avx512_vfmadd_f32;
3623 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
3624 Rep = Builder.CreateCall(FMA, Ops);
3625 } else {
3626 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3627 Intrinsic::fma,
3628 A->getType());
3629 Rep = Builder.CreateCall(FMA, { A, B, C });
3630 }
3631
3632 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) :
3633 IsMask3 ? C : A;
3634
3635 // For Mask3 with NegAcc, we need to create a new extractelement that
3636 // avoids the negation above.
3637 if (NegAcc && IsMask3)
3638 PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
3639 (uint64_t)0);
3640
3641 Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
3642 Rep, PassThru);
3643 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
3644 Rep, (uint64_t)0);
3645 } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.p") ||
3646 Name.startswith("avx512.mask.vfnmadd.p") ||
3647 Name.startswith("avx512.mask.vfnmsub.p") ||
3648 Name.startswith("avx512.mask3.vfmadd.p") ||
3649 Name.startswith("avx512.mask3.vfmsub.p") ||
3650 Name.startswith("avx512.mask3.vfnmsub.p") ||
3651 Name.startswith("avx512.maskz.vfmadd.p"))) {
3652 bool IsMask3 = Name[11] == '3';
3653 bool IsMaskZ = Name[11] == 'z';
3654 // Drop the "avx512.mask." to make it easier.
3655 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3656 bool NegMul = Name[2] == 'n';
3657 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3658
3659 Value *A = CI->getArgOperand(0);
3660 Value *B = CI->getArgOperand(1);
3661 Value *C = CI->getArgOperand(2);
3662
3663 if (NegMul && (IsMask3 || IsMaskZ))
3664 A = Builder.CreateFNeg(A);
3665 if (NegMul && !(IsMask3 || IsMaskZ))
3666 B = Builder.CreateFNeg(B);
3667 if (NegAcc)
3668 C = Builder.CreateFNeg(C);
3669
3670 if (CI->arg_size() == 5 &&
3671 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3672 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
3673 Intrinsic::ID IID;
3674 // Check the character before ".512" in string.
3675 if (Name[Name.size()-5] == 's')
3676 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3677 else
3678 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3679
3680 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3681 { A, B, C, CI->getArgOperand(4) });
3682 } else {
3683 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3684 Intrinsic::fma,
3685 A->getType());
3686 Rep = Builder.CreateCall(FMA, { A, B, C });
3687 }
3688
3689 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3690 IsMask3 ? CI->getArgOperand(2) :
3691 CI->getArgOperand(0);
3692
3693 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3694 } else if (IsX86 && Name.startswith("fma.vfmsubadd.p")) {
3695 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3696 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3697 Intrinsic::ID IID;
3698 if (VecWidth == 128 && EltWidth == 32)
3699 IID = Intrinsic::x86_fma_vfmaddsub_ps;
3700 else if (VecWidth == 256 && EltWidth == 32)
3701 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3702 else if (VecWidth == 128 && EltWidth == 64)
3703 IID = Intrinsic::x86_fma_vfmaddsub_pd;
3704 else if (VecWidth == 256 && EltWidth == 64)
3705 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3706 else
3707 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 3707);
3708
3709 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3710 CI->getArgOperand(2) };
3711 Ops[2] = Builder.CreateFNeg(Ops[2]);
3712 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3713 Ops);
3714 } else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") ||
3715 Name.startswith("avx512.mask3.vfmaddsub.p") ||
3716 Name.startswith("avx512.maskz.vfmaddsub.p") ||
3717 Name.startswith("avx512.mask3.vfmsubadd.p"))) {
3718 bool IsMask3 = Name[11] == '3';
3719 bool IsMaskZ = Name[11] == 'z';
3720 // Drop the "avx512.mask." to make it easier.
3721 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3722 bool IsSubAdd = Name[3] == 's';
3723 if (CI->arg_size() == 5) {
3724 Intrinsic::ID IID;
3725 // Check the character before ".512" in string.
3726 if (Name[Name.size()-5] == 's')
3727 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3728 else
3729 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3730
3731 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3732 CI->getArgOperand(2), CI->getArgOperand(4) };
3733 if (IsSubAdd)
3734 Ops[2] = Builder.CreateFNeg(Ops[2]);
3735
3736 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3737 Ops);
3738 } else {
3739 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3740
3741 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3742 CI->getArgOperand(2) };
3743
3744 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3745 Ops[0]->getType());
3746 Value *Odd = Builder.CreateCall(FMA, Ops);
3747 Ops[2] = Builder.CreateFNeg(Ops[2]);
3748 Value *Even = Builder.CreateCall(FMA, Ops);
3749
3750 if (IsSubAdd)
3751 std::swap(Even, Odd);
3752
3753 SmallVector<int, 32> Idxs(NumElts);
3754 for (int i = 0; i != NumElts; ++i)
3755 Idxs[i] = i + (i % 2) * NumElts;
3756
3757 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
3758 }
3759
3760 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3761 IsMask3 ? CI->getArgOperand(2) :
3762 CI->getArgOperand(0);
3763
3764 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3765 } else if (IsX86 && (Name.startswith("avx512.mask.pternlog.") ||
3766 Name.startswith("avx512.maskz.pternlog."))) {
3767 bool ZeroMask = Name[11] == 'z';
3768 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3769 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3770 Intrinsic::ID IID;
3771 if (VecWidth == 128 && EltWidth == 32)
3772 IID = Intrinsic::x86_avx512_pternlog_d_128;
3773 else if (VecWidth == 256 && EltWidth == 32)
3774 IID = Intrinsic::x86_avx512_pternlog_d_256;
3775 else if (VecWidth == 512 && EltWidth == 32)
3776 IID = Intrinsic::x86_avx512_pternlog_d_512;
3777 else if (VecWidth == 128 && EltWidth == 64)
3778 IID = Intrinsic::x86_avx512_pternlog_q_128;
3779 else if (VecWidth == 256 && EltWidth == 64)
3780 IID = Intrinsic::x86_avx512_pternlog_q_256;
3781 else if (VecWidth == 512 && EltWidth == 64)
3782 IID = Intrinsic::x86_avx512_pternlog_q_512;
3783 else
3784 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 3784);
3785
3786 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3787 CI->getArgOperand(2), CI->getArgOperand(3) };
3788 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3789 Args);
3790 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3791 : CI->getArgOperand(0);
3792 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
3793 } else if (IsX86 && (Name.startswith("avx512.mask.vpmadd52") ||
3794 Name.startswith("avx512.maskz.vpmadd52"))) {
3795 bool ZeroMask = Name[11] == 'z';
3796 bool High = Name[20] == 'h' || Name[21] == 'h';
3797 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3798 Intrinsic::ID IID;
3799 if (VecWidth == 128 && !High)
3800 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3801 else if (VecWidth == 256 && !High)
3802 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3803 else if (VecWidth == 512 && !High)
3804 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3805 else if (VecWidth == 128 && High)
3806 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3807 else if (VecWidth == 256 && High)
3808 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3809 else if (VecWidth == 512 && High)
3810 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3811 else
3812 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 3812);
3813
3814 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3815 CI->getArgOperand(2) };
3816 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3817 Args);
3818 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3819 : CI->getArgOperand(0);
3820 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3821 } else if (IsX86 && (Name.startswith("avx512.mask.vpermi2var.") ||
3822 Name.startswith("avx512.mask.vpermt2var.") ||
3823 Name.startswith("avx512.maskz.vpermt2var."))) {
3824 bool ZeroMask = Name[11] == 'z';
3825 bool IndexForm = Name[17] == 'i';
3826 Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
3827 } else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") ||
3828 Name.startswith("avx512.maskz.vpdpbusd.") ||
3829 Name.startswith("avx512.mask.vpdpbusds.") ||
3830 Name.startswith("avx512.maskz.vpdpbusds."))) {
3831 bool ZeroMask = Name[11] == 'z';
3832 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3833 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3834 Intrinsic::ID IID;
3835 if (VecWidth == 128 && !IsSaturating)
3836 IID = Intrinsic::x86_avx512_vpdpbusd_128;
3837 else if (VecWidth == 256 && !IsSaturating)
3838 IID = Intrinsic::x86_avx512_vpdpbusd_256;
3839 else if (VecWidth == 512 && !IsSaturating)
3840 IID = Intrinsic::x86_avx512_vpdpbusd_512;
3841 else if (VecWidth == 128 && IsSaturating)
3842 IID = Intrinsic::x86_avx512_vpdpbusds_128;
3843 else if (VecWidth == 256 && IsSaturating)
3844 IID = Intrinsic::x86_avx512_vpdpbusds_256;
3845 else if (VecWidth == 512 && IsSaturating)
3846 IID = Intrinsic::x86_avx512_vpdpbusds_512;
3847 else
3848 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 3848);
3849
3850 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3851 CI->getArgOperand(2) };
3852 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3853 Args);
3854 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3855 : CI->getArgOperand(0);
3856 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3857 } else if (IsX86 && (Name.startswith("avx512.mask.vpdpwssd.") ||
3858 Name.startswith("avx512.maskz.vpdpwssd.") ||
3859 Name.startswith("avx512.mask.vpdpwssds.") ||
3860 Name.startswith("avx512.maskz.vpdpwssds."))) {
3861 bool ZeroMask = Name[11] == 'z';
3862 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3863 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3864 Intrinsic::ID IID;
3865 if (VecWidth == 128 && !IsSaturating)
3866 IID = Intrinsic::x86_avx512_vpdpwssd_128;
3867 else if (VecWidth == 256 && !IsSaturating)
3868 IID = Intrinsic::x86_avx512_vpdpwssd_256;
3869 else if (VecWidth == 512 && !IsSaturating)
3870 IID = Intrinsic::x86_avx512_vpdpwssd_512;
3871 else if (VecWidth == 128 && IsSaturating)
3872 IID = Intrinsic::x86_avx512_vpdpwssds_128;
3873 else if (VecWidth == 256 && IsSaturating)
3874 IID = Intrinsic::x86_avx512_vpdpwssds_256;
3875 else if (VecWidth == 512 && IsSaturating)
3876 IID = Intrinsic::x86_avx512_vpdpwssds_512;
3877 else
3878 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 3878);
3879
3880 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3881 CI->getArgOperand(2) };
3882 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3883 Args);
3884 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3885 : CI->getArgOperand(0);
3886 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3887 } else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
3888 Name == "addcarry.u32" || Name == "addcarry.u64" ||
3889 Name == "subborrow.u32" || Name == "subborrow.u64")) {
3890 Intrinsic::ID IID;
3891 if (Name[0] == 'a' && Name.back() == '2')
3892 IID = Intrinsic::x86_addcarry_32;
3893 else if (Name[0] == 'a' && Name.back() == '4')
3894 IID = Intrinsic::x86_addcarry_64;
3895 else if (Name[0] == 's' && Name.back() == '2')
3896 IID = Intrinsic::x86_subborrow_32;
3897 else if (Name[0] == 's' && Name.back() == '4')
3898 IID = Intrinsic::x86_subborrow_64;
3899 else
3900 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "llvm/lib/IR/AutoUpgrade.cpp"
, 3900);
3901
3902 // Make a call with 3 operands.
3903 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3904 CI->getArgOperand(2)};
3905 Value *NewCall = Builder.CreateCall(
3906 Intrinsic::getDeclaration(CI->getModule(), IID),
3907 Args);
3908
3909 // Extract the second result and store it.
3910 Value *Data = Builder.CreateExtractValue(NewCall, 1);
3911 // Cast the pointer to the right type.
3912 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
3913 llvm::PointerType::getUnqual(Data->getType()));
3914 Builder.CreateAlignedStore(Data, Ptr, Align(1));
3915 // Replace the original call result with the first result of the new call.
3916 Value *CF = Builder.CreateExtractValue(NewCall, 0);
3917
3918 CI->replaceAllUsesWith(CF);
3919 Rep = nullptr;
3920 } else if (IsX86 && Name.startswith("avx512.mask.") &&
3921 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
3922 // Rep will be updated by the call in the condition.
3923 } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {
3924 Value *Arg = CI->getArgOperand(0);
3925 Value *Neg = Builder.CreateNeg(Arg, "neg");
3926 Value *Cmp = Builder.CreateICmpSGE(
3927 Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
3928 Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
3929 } else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") ||
3930 Name.startswith("atomic.load.add.f64.p"))) {
3931 Value *Ptr = CI->getArgOperand(0);
3932 Value *Val = CI->getArgOperand(1);
3933 Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(),
3934 AtomicOrdering::SequentiallyConsistent);
3935 } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
3936 Name == "max.ui" || Name == "max.ull")) {
3937 Value *Arg0 = CI->getArgOperand(0);
3938 Value *Arg1 = CI->getArgOperand(1);
3939 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3940 ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
3941 : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
3942 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
3943 } else if (IsNVVM && (Name == "min.i" || Name == "min.ll" ||
3944 Name == "min.ui" || Name == "min.ull")) {
3945 Value *Arg0 = CI->getArgOperand(0);
3946 Value *Arg1 = CI->getArgOperand(1);
3947 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3948 ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
3949 : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
3950 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
3951 } else if (IsNVVM && Name == "clz.ll") {
3952 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64.
3953 Value *Arg = CI->getArgOperand(0);
3954 Value *Ctlz = Builder.CreateCall(
3955 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
3956 {Arg->getType()}),
3957 {Arg, Builder.getFalse()}, "ctlz");
3958 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3959 } else if (IsNVVM && Name == "popc.ll") {
3960 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an
3961 // i64.
3962 Value *Arg = CI->getArgOperand(0);
3963 Value *Popc = Builder.CreateCall(
3964 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
3965 {Arg->getType()}),
3966 Arg, "ctpop");
3967 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3968 } else if (IsNVVM && Name == "h2f") {
3969 Rep = Builder.CreateCall(Intrinsic::getDeclaration(
3970 F->getParent(), Intrinsic::convert_from_fp16,
3971 {Builder.getFloatTy()}),
3972 CI->getArgOperand(0), "h2f");
3973 } else if (IsARM) {
3974 Rep = UpgradeARMIntrinsicCall(Name, CI, F, Builder);
3975 } else {
3976 llvm_unreachable("Unknown function for CallBase upgrade.")::llvm::llvm_unreachable_internal("Unknown function for CallBase upgrade."
, "llvm/lib/IR/AutoUpgrade.cpp", 3976);
3977 }
3978
3979 if (Rep)
3980 CI->replaceAllUsesWith(Rep);
3981 CI->eraseFromParent();
3982 return;
3983 }
3984
3985 const auto &DefaultCase = [&]() -> void {
3986 if (CI->getFunctionType() == NewFn->getFunctionType()) {
3987 // Handle generic mangling change.
3988 assert((static_cast <bool> ((CI->getCalledFunction()->getName
() != NewFn->getName()) && "Unknown function for CallBase upgrade and isn't just a name change"
) ? void (0) : __assert_fail ("(CI->getCalledFunction()->getName() != NewFn->getName()) && \"Unknown function for CallBase upgrade and isn't just a name change\""
, "llvm/lib/IR/AutoUpgrade.cpp", 3990, __extension__ __PRETTY_FUNCTION__
))
3989 (CI->getCalledFunction()->getName() != NewFn->getName()) &&(static_cast <bool> ((CI->getCalledFunction()->getName
() != NewFn->getName()) && "Unknown function for CallBase upgrade and isn't just a name change"
) ? void (0) : __assert_fail ("(CI->getCalledFunction()->getName() != NewFn->getName()) && \"Unknown function for CallBase upgrade and isn't just a name change\""
, "llvm/lib/IR/AutoUpgrade.cpp", 3990, __extension__ __PRETTY_FUNCTION__
))
3990 "Unknown function for CallBase upgrade and isn't just a name change")(static_cast <bool> ((CI->getCalledFunction()->getName
() != NewFn->getName()) && "Unknown function for CallBase upgrade and isn't just a name change"
) ? void (0) : __assert_fail ("(CI->getCalledFunction()->getName() != NewFn->getName()) && \"Unknown function for CallBase upgrade and isn't just a name change\""
, "llvm/lib/IR/AutoUpgrade.cpp", 3990, __extension__ __PRETTY_FUNCTION__
));
3991 CI->setCalledFunction(NewFn);
3992 return;
3993 }
3994
3995 // This must be an upgrade from a named to a literal struct.
3996 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
3997 assert(OldST != NewFn->getReturnType() &&(static_cast <bool> (OldST != NewFn->getReturnType()
&& "Return type must have changed") ? void (0) : __assert_fail
("OldST != NewFn->getReturnType() && \"Return type must have changed\""
, "llvm/lib/IR/AutoUpgrade.cpp", 3998, __extension__ __PRETTY_FUNCTION__
))
3998 "Return type must have changed")(static_cast <bool> (OldST != NewFn->getReturnType()
&& "Return type must have changed") ? void (0) : __assert_fail
("OldST != NewFn->getReturnType() && \"Return type must have changed\""
, "llvm/lib/IR/AutoUpgrade.cpp", 3998, __extension__ __PRETTY_FUNCTION__
));
3999 assert(OldST->getNumElements() ==(static_cast <bool> (OldST->getNumElements() == cast
<StructType>(NewFn->getReturnType())->getNumElements
() && "Must have same number of elements") ? void (0)
: __assert_fail ("OldST->getNumElements() == cast<StructType>(NewFn->getReturnType())->getNumElements() && \"Must have same number of elements\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4001, __extension__ __PRETTY_FUNCTION__
))
4000 cast<StructType>(NewFn->getReturnType())->getNumElements() &&(static_cast <bool> (OldST->getNumElements() == cast
<StructType>(NewFn->getReturnType())->getNumElements
() && "Must have same number of elements") ? void (0)
: __assert_fail ("OldST->getNumElements() == cast<StructType>(NewFn->getReturnType())->getNumElements() && \"Must have same number of elements\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4001, __extension__ __PRETTY_FUNCTION__
))
4001 "Must have same number of elements")(static_cast <bool> (OldST->getNumElements() == cast
<StructType>(NewFn->getReturnType())->getNumElements
() && "Must have same number of elements") ? void (0)
: __assert_fail ("OldST->getNumElements() == cast<StructType>(NewFn->getReturnType())->getNumElements() && \"Must have same number of elements\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4001, __extension__ __PRETTY_FUNCTION__
));
4002
4003 SmallVector<Value *> Args(CI->args());
4004 Value *NewCI = Builder.CreateCall(NewFn, Args);
4005 Value *Res = PoisonValue::get(OldST);
4006 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
4007 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
4008 Res = Builder.CreateInsertValue(Res, Elem, Idx);
4009 }
4010 CI->replaceAllUsesWith(Res);
4011 CI->eraseFromParent();
4012 return;
4013 }
4014
4015 // We're probably about to produce something invalid. Let the verifier catch
4016 // it instead of dying here.
4017 CI->setCalledOperand(
4018 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
4019 return;
4020 };
4021 CallInst *NewCall = nullptr;
4022 switch (NewFn->getIntrinsicID()) {
4023 default: {
4024 DefaultCase();
4025 return;
4026 }
4027 case Intrinsic::arm_neon_vst1:
4028 case Intrinsic::arm_neon_vst2:
4029 case Intrinsic::arm_neon_vst3:
4030 case Intrinsic::arm_neon_vst4:
4031 case Intrinsic::arm_neon_vst2lane:
4032 case Intrinsic::arm_neon_vst3lane:
4033 case Intrinsic::arm_neon_vst4lane: {
4034 SmallVector<Value *, 4> Args(CI->args());
4035 NewCall = Builder.CreateCall(NewFn, Args);
4036 break;
4037 }
4038 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
4039 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
4040 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
4041 LLVMContext &Ctx = F->getParent()->getContext();
4042 SmallVector<Value *, 4> Args(CI->args());
4043 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
4044 cast<ConstantInt>(Args[3])->getZExtValue());
4045 NewCall = Builder.CreateCall(NewFn, Args);
4046 break;
4047 }
4048 case Intrinsic::aarch64_sve_ld3_sret:
4049 case Intrinsic::aarch64_sve_ld4_sret:
4050 case Intrinsic::aarch64_sve_ld2_sret: {
4051 StringRef Name = F->getName();
4052 Name = Name.substr(5);
4053 unsigned N = StringSwitch<unsigned>(Name)
4054 .StartsWith("aarch64.sve.ld2", 2)
4055 .StartsWith("aarch64.sve.ld3", 3)
4056 .StartsWith("aarch64.sve.ld4", 4)
4057 .Default(0);
4058 ScalableVectorType *RetTy =
4059 dyn_cast<ScalableVectorType>(F->getReturnType());
4060 unsigned MinElts = RetTy->getMinNumElements() / N;
4061 SmallVector<Value *, 2> Args(CI->args());
4062 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
4063 Value *Ret = llvm::PoisonValue::get(RetTy);
4064 for (unsigned I = 0; I < N; I++) {
4065 Value *Idx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
4066 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
4067 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
4068 }
4069 NewCall = dyn_cast<CallInst>(Ret);
4070 break;
4071 }
4072
4073 case Intrinsic::vector_extract: {
4074 StringRef Name = F->getName();
4075 Name = Name.substr(5); // Strip llvm
4076 if (!Name.startswith("aarch64.sve.tuple.get")) {
4077 DefaultCase();
4078 return;
4079 }
4080 ScalableVectorType *RetTy =
4081 dyn_cast<ScalableVectorType>(F->getReturnType());
4082 unsigned MinElts = RetTy->getMinNumElements();
4083 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4084 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
4085 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
4086 break;
4087 }
4088
4089 case Intrinsic::vector_insert: {
4090 StringRef Name = F->getName();
4091 Name = Name.substr(5);
4092 if (!Name.startswith("aarch64.sve.tuple")) {
4093 DefaultCase();
4094 return;
4095 }
4096 if (Name.startswith("aarch64.sve.tuple.set")) {
4097 unsigned I = dyn_cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4098 ScalableVectorType *Ty =
4099 dyn_cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
4100 Value *NewIdx =
4101 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
4102 NewCall = Builder.CreateCall(
4103 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
4104 break;
4105 }
4106 if (Name.startswith("aarch64.sve.tuple.create")) {
4107 unsigned N = StringSwitch<unsigned>(Name)
4108 .StartsWith("aarch64.sve.tuple.create2", 2)
4109 .StartsWith("aarch64.sve.tuple.create3", 3)
4110 .StartsWith("aarch64.sve.tuple.create4", 4)
4111 .Default(0);
4112 assert(N > 1 && "Create is expected to be between 2-4")(static_cast <bool> (N > 1 && "Create is expected to be between 2-4"
) ? void (0) : __assert_fail ("N > 1 && \"Create is expected to be between 2-4\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4112, __extension__ __PRETTY_FUNCTION__
));
4113 ScalableVectorType *RetTy =
4114 dyn_cast<ScalableVectorType>(F->getReturnType());
4115 Value *Ret = llvm::PoisonValue::get(RetTy);
4116 unsigned MinElts = RetTy->getMinNumElements() / N;
4117 for (unsigned I = 0; I < N; I++) {
4118 Value *Idx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
4119 Value *V = CI->getArgOperand(I);
4120 Ret = Builder.CreateInsertVector(RetTy, Ret, V, Idx);
4121 }
4122 NewCall = dyn_cast<CallInst>(Ret);
4123 }
4124 break;
4125 }
4126
4127 case Intrinsic::arm_neon_bfdot:
4128 case Intrinsic::arm_neon_bfmmla:
4129 case Intrinsic::arm_neon_bfmlalb:
4130 case Intrinsic::arm_neon_bfmlalt:
4131 case Intrinsic::aarch64_neon_bfdot:
4132 case Intrinsic::aarch64_neon_bfmmla:
4133 case Intrinsic::aarch64_neon_bfmlalb:
4134 case Intrinsic::aarch64_neon_bfmlalt: {
4135 SmallVector<Value *, 3> Args;
4136 assert(CI->arg_size() == 3 &&(static_cast <bool> (CI->arg_size() == 3 && "Mismatch between function args and call args"
) ? void (0) : __assert_fail ("CI->arg_size() == 3 && \"Mismatch between function args and call args\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4137, __extension__ __PRETTY_FUNCTION__
))
4137 "Mismatch between function args and call args")(static_cast <bool> (CI->arg_size() == 3 && "Mismatch between function args and call args"
) ? void (0) : __assert_fail ("CI->arg_size() == 3 && \"Mismatch between function args and call args\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4137, __extension__ __PRETTY_FUNCTION__
));
4138 size_t OperandWidth =
4139 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
4140 assert((OperandWidth == 64 || OperandWidth == 128) &&(static_cast <bool> ((OperandWidth == 64 || OperandWidth
== 128) && "Unexpected operand width") ? void (0) : __assert_fail
("(OperandWidth == 64 || OperandWidth == 128) && \"Unexpected operand width\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4141, __extension__ __PRETTY_FUNCTION__
))
4141 "Unexpected operand width")(static_cast <bool> ((OperandWidth == 64 || OperandWidth
== 128) && "Unexpected operand width") ? void (0) : __assert_fail
("(OperandWidth == 64 || OperandWidth == 128) && \"Unexpected operand width\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4141, __extension__ __PRETTY_FUNCTION__
));
4142 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
4143 auto Iter = CI->args().begin();
4144 Args.push_back(*Iter++);
4145 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
4146 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
4147 NewCall = Builder.CreateCall(NewFn, Args);
4148 break;
4149 }
4150
4151 case Intrinsic::bitreverse:
4152 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
4153 break;
4154
4155 case Intrinsic::ctlz:
4156 case Intrinsic::cttz:
4157 assert(CI->arg_size() == 1 &&(static_cast <bool> (CI->arg_size() == 1 && "Mismatch between function args and call args"
) ? void (0) : __assert_fail ("CI->arg_size() == 1 && \"Mismatch between function args and call args\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4158, __extension__ __PRETTY_FUNCTION__
))
4158 "Mismatch between function args and call args")(static_cast <bool> (CI->arg_size() == 1 && "Mismatch between function args and call args"
) ? void (0) : __assert_fail ("CI->arg_size() == 1 && \"Mismatch between function args and call args\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4158, __extension__ __PRETTY_FUNCTION__
));
4159 NewCall =
4160 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
4161 break;
4162
4163 case Intrinsic::objectsize: {
4164 Value *NullIsUnknownSize =
4165 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
4166 Value *Dynamic =
4167 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
4168 NewCall = Builder.CreateCall(
4169 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
4170 break;
4171 }
4172
4173 case Intrinsic::ctpop:
4174 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
4175 break;
4176
4177 case Intrinsic::convert_from_fp16:
4178 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
4179 break;
4180
4181 case Intrinsic::dbg_value: {
4182 StringRef Name = F->getName();
4183 Name = Name.substr(5); // Strip llvm.
4184 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
4185 if (Name.startswith("dbg.addr")) {
4186 DIExpression *Expr = cast<DIExpression>(
4187 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
4188 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
4189 NewCall =
4190 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
4191 MetadataAsValue::get(C, Expr)});
4192 break;
4193 }
4194
4195 // Upgrade from the old version that had an extra offset argument.
4196 assert(CI->arg_size() == 4)(static_cast <bool> (CI->arg_size() == 4) ? void (0)
: __assert_fail ("CI->arg_size() == 4", "llvm/lib/IR/AutoUpgrade.cpp"
, 4196, __extension__ __PRETTY_FUNCTION__));
4197 // Drop nonzero offsets instead of attempting to upgrade them.
4198 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
4199 if (Offset->isZeroValue()) {
4200 NewCall = Builder.CreateCall(
4201 NewFn,
4202 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
4203 break;
4204 }
4205 CI->eraseFromParent();
4206 return;
4207 }
4208
4209 case Intrinsic::ptr_annotation:
4210 // Upgrade from versions that lacked the annotation attribute argument.
4211 if (CI->arg_size() != 4) {
4212 DefaultCase();
4213 return;
4214 }
4215
4216 // Create a new call with an added null annotation attribute argument.
4217 NewCall = Builder.CreateCall(
4218 NewFn,
4219 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
4220 CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});
4221 NewCall->takeName(CI);
4222 CI->replaceAllUsesWith(NewCall);
4223 CI->eraseFromParent();
4224 return;
4225
4226 case Intrinsic::var_annotation:
4227 // Upgrade from versions that lacked the annotation attribute argument.
4228 if (CI->arg_size() != 4) {
4229 DefaultCase();
4230 return;
4231 }
4232 // Create a new call with an added null annotation attribute argument.
4233 NewCall = Builder.CreateCall(
4234 NewFn,
4235 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
4236 CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});
4237 NewCall->takeName(CI);
4238 CI->replaceAllUsesWith(NewCall);
4239 CI->eraseFromParent();
4240 return;
4241
4242 case Intrinsic::x86_xop_vfrcz_ss:
4243 case Intrinsic::x86_xop_vfrcz_sd:
4244 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
4245 break;
4246
4247 case Intrinsic::x86_xop_vpermil2pd:
4248 case Intrinsic::x86_xop_vpermil2ps:
4249 case Intrinsic::x86_xop_vpermil2pd_256:
4250 case Intrinsic::x86_xop_vpermil2ps_256: {
4251 SmallVector<Value *, 4> Args(CI->args());
4252 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
4253 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
4254 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
4255 NewCall = Builder.CreateCall(NewFn, Args);
4256 break;
4257 }
4258
4259 case Intrinsic::x86_sse41_ptestc:
4260 case Intrinsic::x86_sse41_ptestz:
4261 case Intrinsic::x86_sse41_ptestnzc: {
4262 // The arguments for these intrinsics used to be v4f32, and changed
4263 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
4264 // So, the only thing required is a bitcast for both arguments.
4265 // First, check the arguments have the old type.
4266 Value *Arg0 = CI->getArgOperand(0);
4267 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
4268 return;
4269
4270 // Old intrinsic, add bitcasts
4271 Value *Arg1 = CI->getArgOperand(1);
4272
4273 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
4274
4275 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
4276 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
4277
4278 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
4279 break;
4280 }
4281
4282 case Intrinsic::x86_rdtscp: {
4283 // This used to take 1 arguments. If we have no arguments, it is already
4284 // upgraded.
4285 if (CI->getNumOperands() == 0)
4286 return;
4287
4288 NewCall = Builder.CreateCall(NewFn);
4289 // Extract the second result and store it.
4290 Value *Data = Builder.CreateExtractValue(NewCall, 1);
4291 // Cast the pointer to the right type.
4292 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
4293 llvm::PointerType::getUnqual(Data->getType()));
4294 Builder.CreateAlignedStore(Data, Ptr, Align(1));
4295 // Replace the original call result with the first result of the new call.
4296 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
4297
4298 NewCall->takeName(CI);
4299 CI->replaceAllUsesWith(TSC);
4300 CI->eraseFromParent();
4301 return;
4302 }
4303
4304 case Intrinsic::x86_sse41_insertps:
4305 case Intrinsic::x86_sse41_dppd:
4306 case Intrinsic::x86_sse41_dpps:
4307 case Intrinsic::x86_sse41_mpsadbw:
4308 case Intrinsic::x86_avx_dp_ps_256:
4309 case Intrinsic::x86_avx2_mpsadbw: {
4310 // Need to truncate the last argument from i32 to i8 -- this argument models
4311 // an inherently 8-bit immediate operand to these x86 instructions.
4312 SmallVector<Value *, 4> Args(CI->args());
4313
4314 // Replace the last argument with a trunc.
4315 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
4316 NewCall = Builder.CreateCall(NewFn, Args);
4317 break;
4318 }
4319
4320 case Intrinsic::x86_avx512_mask_cmp_pd_128:
4321 case Intrinsic::x86_avx512_mask_cmp_pd_256:
4322 case Intrinsic::x86_avx512_mask_cmp_pd_512:
4323 case Intrinsic::x86_avx512_mask_cmp_ps_128:
4324 case Intrinsic::x86_avx512_mask_cmp_ps_256:
4325 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
4326 SmallVector<Value *, 4> Args(CI->args());
4327 unsigned NumElts =
4328 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
4329 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
4330
4331 NewCall = Builder.CreateCall(NewFn, Args);
4332 Value *Res = ApplyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
4333
4334 NewCall->takeName(CI);
4335 CI->replaceAllUsesWith(Res);
4336 CI->eraseFromParent();
4337 return;
4338 }
4339
4340 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
4341 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
4342 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
4343 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
4344 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
4345 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
4346 SmallVector<Value *, 4> Args(CI->args());
4347 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4348 if (NewFn->getIntrinsicID() ==
4349 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
4350 Args[1] = Builder.CreateBitCast(
4351 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
4352
4353 NewCall = Builder.CreateCall(NewFn, Args);
4354 Value *Res = Builder.CreateBitCast(
4355 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
4356
4357 NewCall->takeName(CI);
4358 CI->replaceAllUsesWith(Res);
4359 CI->eraseFromParent();
4360 return;
4361 }
4362 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
4363 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
4364 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
4365 SmallVector<Value *, 4> Args(CI->args());
4366 unsigned NumElts =
4367 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
4368 Args[1] = Builder.CreateBitCast(
4369 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
4370 Args[2] = Builder.CreateBitCast(
4371 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
4372
4373 NewCall = Builder.CreateCall(NewFn, Args);
4374 break;
4375 }
4376
4377 case Intrinsic::thread_pointer: {
4378 NewCall = Builder.CreateCall(NewFn, {});
4379 break;
4380 }
4381
4382 case Intrinsic::invariant_start:
4383 case Intrinsic::invariant_end: {
4384 SmallVector<Value *, 4> Args(CI->args());
4385 NewCall = Builder.CreateCall(NewFn, Args);
4386 break;
4387 }
4388 case Intrinsic::masked_load:
4389 case Intrinsic::masked_store:
4390 case Intrinsic::masked_gather:
4391 case Intrinsic::masked_scatter: {
4392 SmallVector<Value *, 4> Args(CI->args());
4393 NewCall = Builder.CreateCall(NewFn, Args);
4394 NewCall->copyMetadata(*CI);
4395 break;
4396 }
4397
4398 case Intrinsic::memcpy:
4399 case Intrinsic::memmove:
4400 case Intrinsic::memset: {
4401 // We have to make sure that the call signature is what we're expecting.
4402 // We only want to change the old signatures by removing the alignment arg:
4403 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
4404 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
4405 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
4406 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
4407 // Note: i8*'s in the above can be any pointer type
4408 if (CI->arg_size() != 5) {
4409 DefaultCase();
4410 return;
4411 }
4412 // Remove alignment argument (3), and add alignment attributes to the
4413 // dest/src pointers.
4414 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
4415 CI->getArgOperand(2), CI->getArgOperand(4)};
4416 NewCall = Builder.CreateCall(NewFn, Args);
4417 AttributeList OldAttrs = CI->getAttributes();
4418 AttributeList NewAttrs = AttributeList::get(
4419 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
4420 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
4421 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
4422 NewCall->setAttributes(NewAttrs);
4423 auto *MemCI = cast<MemIntrinsic>(NewCall);
4424 // All mem intrinsics support dest alignment.
4425 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
4426 MemCI->setDestAlignment(Align->getMaybeAlignValue());
4427 // Memcpy/Memmove also support source alignment.
4428 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
4429 MTI->setSourceAlignment(Align->getMaybeAlignValue());
4430 break;
4431 }
4432 }
4433 assert(NewCall && "Should have either set this variable or returned through "(static_cast <bool> (NewCall && "Should have either set this variable or returned through "
"the default case") ? void (0) : __assert_fail ("NewCall && \"Should have either set this variable or returned through \" \"the default case\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4434, __extension__ __PRETTY_FUNCTION__
))
4434 "the default case")(static_cast <bool> (NewCall && "Should have either set this variable or returned through "
"the default case") ? void (0) : __assert_fail ("NewCall && \"Should have either set this variable or returned through \" \"the default case\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4434, __extension__ __PRETTY_FUNCTION__
));
4435 NewCall->takeName(CI);
4436 CI->replaceAllUsesWith(NewCall);
4437 CI->eraseFromParent();
4438}
4439
4440void llvm::UpgradeCallsToIntrinsic(Function *F) {
4441 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.")(static_cast <bool> (F && "Illegal attempt to upgrade a non-existent intrinsic."
) ? void (0) : __assert_fail ("F && \"Illegal attempt to upgrade a non-existent intrinsic.\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4441, __extension__ __PRETTY_FUNCTION__
));
4442
4443 // Check if this function should be upgraded and get the replacement function
4444 // if there is one.
4445 Function *NewFn;
4446 if (UpgradeIntrinsicFunction(F, NewFn)) {
4447 // Replace all users of the old function with the new function or new
4448 // instructions. This is not a range loop because the call is deleted.
4449 for (User *U : make_early_inc_range(F->users()))
4450 if (CallBase *CB = dyn_cast<CallBase>(U))
4451 UpgradeIntrinsicCall(CB, NewFn);
4452
4453 // Remove old function, no longer used, from the module.
4454 F->eraseFromParent();
4455 }
4456}
4457
4458MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
4459 // Check if the tag uses struct-path aware TBAA format.
4460 if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3)
4461 return &MD;
4462
4463 auto &Context = MD.getContext();
4464 if (MD.getNumOperands() == 3) {
4465 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
4466 MDNode *ScalarType = MDNode::get(Context, Elts);
4467 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
4468 Metadata *Elts2[] = {ScalarType, ScalarType,
4469 ConstantAsMetadata::get(
4470 Constant::getNullValue(Type::getInt64Ty(Context))),
4471 MD.getOperand(2)};
4472 return MDNode::get(Context, Elts2);
4473 }
4474 // Create a MDNode <MD, MD, offset 0>
4475 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
4476 Type::getInt64Ty(Context)))};
4477 return MDNode::get(Context, Elts);
4478}
4479
4480Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
4481 Instruction *&Temp) {
4482 if (Opc != Instruction::BitCast)
4483 return nullptr;
4484
4485 Temp = nullptr;
4486 Type *SrcTy = V->getType();
4487 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
4488 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
4489 LLVMContext &Context = V->getContext();
4490
4491 // We have no information about target data layout, so we assume that
4492 // the maximum pointer size is 64bit.
4493 Type *MidTy = Type::getInt64Ty(Context);
4494 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
4495
4496 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
4497 }
4498
4499 return nullptr;
4500}
4501
4502Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
4503 if (Opc != Instruction::BitCast)
4504 return nullptr;
4505
4506 Type *SrcTy = C->getType();
4507 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
4508 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
4509 LLVMContext &Context = C->getContext();
4510
4511 // We have no information about target data layout, so we assume that
4512 // the maximum pointer size is 64bit.
4513 Type *MidTy = Type::getInt64Ty(Context);
4514
4515 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
4516 DestTy);
4517 }
4518
4519 return nullptr;
4520}
4521
4522/// Check the debug info version number, if it is out-dated, drop the debug
4523/// info. Return true if module is modified.
4524bool llvm::UpgradeDebugInfo(Module &M) {
4525 if (DisableAutoUpgradeDebugInfo)
4526 return false;
4527
4528 unsigned Version = getDebugMetadataVersionFromModule(M);
4529 if (Version == DEBUG_METADATA_VERSION) {
4530 bool BrokenDebugInfo = false;
4531 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
4532 report_fatal_error("Broken module found, compilation aborted!");
4533 if (!BrokenDebugInfo)
4534 // Everything is ok.
4535 return false;
4536 else {
4537 // Diagnose malformed debug info.
4538 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
4539 M.getContext().diagnose(Diag);
4540 }
4541 }
4542 bool Modified = StripDebugInfo(M);
4543 if (Modified && Version != DEBUG_METADATA_VERSION) {
4544 // Diagnose a version mismatch.
4545 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
4546 M.getContext().diagnose(DiagVersion);
4547 }
4548 return Modified;
4549}
4550
4551/// This checks for objc retain release marker which should be upgraded. It
4552/// returns true if module is modified.
4553static bool UpgradeRetainReleaseMarker(Module &M) {
4554 bool Changed = false;
4555 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
4556 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
4557 if (ModRetainReleaseMarker) {
4558 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
4559 if (Op) {
4560 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
4561 if (ID) {
4562 SmallVector<StringRef, 4> ValueComp;
4563 ID->getString().split(ValueComp, "#");
4564 if (ValueComp.size() == 2) {
4565 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
4566 ID = MDString::get(M.getContext(), NewValue);
4567 }
4568 M.addModuleFlag(Module::Error, MarkerKey, ID);
4569 M.eraseNamedMetadata(ModRetainReleaseMarker);
4570 Changed = true;
4571 }
4572 }
4573 }
4574 return Changed;
4575}
4576
4577void llvm::UpgradeARCRuntime(Module &M) {
4578 // This lambda converts normal function calls to ARC runtime functions to
4579 // intrinsic calls.
4580 auto UpgradeToIntrinsic = [&](const char *OldFunc,
4581 llvm::Intrinsic::ID IntrinsicFunc) {
4582 Function *Fn = M.getFunction(OldFunc);
4583
4584 if (!Fn)
4585 return;
4586
4587 Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);
4588
4589 for (User *U : make_early_inc_range(Fn->users())) {
4590 CallInst *CI = dyn_cast<CallInst>(U);
4591 if (!CI || CI->getCalledFunction() != Fn)
4592 continue;
4593
4594 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
4595 FunctionType *NewFuncTy = NewFn->getFunctionType();
4596 SmallVector<Value *, 2> Args;
4597
4598 // Don't upgrade the intrinsic if it's not valid to bitcast the return
4599 // value to the return type of the old function.
4600 if (NewFuncTy->getReturnType() != CI->getType() &&
4601 !CastInst::castIsValid(Instruction::BitCast, CI,
4602 NewFuncTy->getReturnType()))
4603 continue;
4604
4605 bool InvalidCast = false;
4606
4607 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
4608 Value *Arg = CI->getArgOperand(I);
4609
4610 // Bitcast argument to the parameter type of the new function if it's
4611 // not a variadic argument.
4612 if (I < NewFuncTy->getNumParams()) {
4613 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
4614 // to the parameter type of the new function.
4615 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
4616 NewFuncTy->getParamType(I))) {
4617 InvalidCast = true;
4618 break;
4619 }
4620 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
4621 }
4622 Args.push_back(Arg);
4623 }
4624
4625 if (InvalidCast)
4626 continue;
4627
4628 // Create a call instruction that calls the new function.
4629 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
4630 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4631 NewCall->takeName(CI);
4632
4633 // Bitcast the return value back to the type of the old call.
4634 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
4635
4636 if (!CI->use_empty())
4637 CI->replaceAllUsesWith(NewRetVal);
4638 CI->eraseFromParent();
4639 }
4640
4641 if (Fn->use_empty())
4642 Fn->eraseFromParent();
4643 };
4644
4645 // Unconditionally convert a call to "clang.arc.use" to a call to
4646 // "llvm.objc.clang.arc.use".
4647 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
4648
4649 // Upgrade the retain release marker. If there is no need to upgrade
4650 // the marker, that means either the module is already new enough to contain
4651 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
4652 if (!UpgradeRetainReleaseMarker(M))
4653 return;
4654
4655 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
4656 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
4657 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
4658 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
4659 {"objc_autoreleaseReturnValue",
4660 llvm::Intrinsic::objc_autoreleaseReturnValue},
4661 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
4662 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
4663 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
4664 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
4665 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
4666 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
4667 {"objc_release", llvm::Intrinsic::objc_release},
4668 {"objc_retain", llvm::Intrinsic::objc_retain},
4669 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
4670 {"objc_retainAutoreleaseReturnValue",
4671 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
4672 {"objc_retainAutoreleasedReturnValue",
4673 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
4674 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
4675 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
4676 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
4677 {"objc_unsafeClaimAutoreleasedReturnValue",
4678 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
4679 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
4680 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
4681 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
4682 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
4683 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
4684 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
4685 {"objc_arc_annotation_topdown_bbstart",
4686 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
4687 {"objc_arc_annotation_topdown_bbend",
4688 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
4689 {"objc_arc_annotation_bottomup_bbstart",
4690 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
4691 {"objc_arc_annotation_bottomup_bbend",
4692 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
4693
4694 for (auto &I : RuntimeFuncs)
4695 UpgradeToIntrinsic(I.first, I.second);
4696}
4697
4698bool llvm::UpgradeModuleFlags(Module &M) {
4699 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
4700 if (!ModFlags)
4701 return false;
4702
4703 bool HasObjCFlag = false, HasClassProperties = false, Changed = false;
4704 bool HasSwiftVersionFlag = false;
4705 uint8_t SwiftMajorVersion, SwiftMinorVersion;
4706 uint32_t SwiftABIVersion;
4707 auto Int8Ty = Type::getInt8Ty(M.getContext());
4708 auto Int32Ty = Type::getInt32Ty(M.getContext());
4709
4710 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
4711 MDNode *Op = ModFlags->getOperand(I);
4712 if (Op->getNumOperands() != 3)
4713 continue;
4714 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
4715 if (!ID)
4716 continue;
4717 auto SetBehavior = [&](Module::ModFlagBehavior B) {
4718 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
4719 Type::getInt32Ty(M.getContext()), B)),
4720 MDString::get(M.getContext(), ID->getString()),
4721 Op->getOperand(2)};
4722 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4723 Changed = true;
4724 };
4725
4726 if (ID->getString() == "Objective-C Image Info Version")
4727 HasObjCFlag = true;
4728 if (ID->getString() == "Objective-C Class Properties")
4729 HasClassProperties = true;
4730 // Upgrade PIC from Error/Max to Min.
4731 if (ID->getString() == "PIC Level") {
4732 if (auto *Behavior =
4733 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
4734 uint64_t V = Behavior->getLimitedValue();
4735 if (V == Module::Error || V == Module::Max)
4736 SetBehavior(Module::Min);
4737 }
4738 }
4739 // Upgrade "PIE Level" from Error to Max.
4740 if (ID->getString() == "PIE Level")
4741 if (auto *Behavior =
4742 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
4743 if (Behavior->getLimitedValue() == Module::Error)
4744 SetBehavior(Module::Max);
4745
4746 // Upgrade branch protection and return address signing module flags. The
4747 // module flag behavior for these fields were Error and now they are Min.
4748 if (ID->getString() == "branch-target-enforcement" ||
4749 ID->getString().startswith("sign-return-address")) {
4750 if (auto *Behavior =
4751 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
4752 if (Behavior->getLimitedValue() == Module::Error) {
4753 Type *Int32Ty = Type::getInt32Ty(M.getContext());
4754 Metadata *Ops[3] = {
4755 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
4756 Op->getOperand(1), Op->getOperand(2)};
4757 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4758 Changed = true;
4759 }
4760 }
4761 }
4762
4763 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
4764 // section name so that llvm-lto will not complain about mismatching
4765 // module flags that is functionally the same.
4766 if (ID->getString() == "Objective-C Image Info Section") {
4767 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
4768 SmallVector<StringRef, 4> ValueComp;
4769 Value->getString().split(ValueComp, " ");
4770 if (ValueComp.size() != 1) {
4771 std::string NewValue;
4772 for (auto &S : ValueComp)
4773 NewValue += S.str();
4774 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
4775 MDString::get(M.getContext(), NewValue)};
4776 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4777 Changed = true;
4778 }
4779 }
4780 }
4781
4782 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
4783 // If the higher bits are set, it adds new module flag for swift info.
4784 if (ID->getString() == "Objective-C Garbage Collection") {
4785 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
4786 if (Md) {
4787 assert(Md->getValue() && "Expected non-empty metadata")(static_cast <bool> (Md->getValue() && "Expected non-empty metadata"
) ? void (0) : __assert_fail ("Md->getValue() && \"Expected non-empty metadata\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4787, __extension__ __PRETTY_FUNCTION__
));
4788 auto Type = Md->getValue()->getType();
4789 if (Type == Int8Ty)
4790 continue;
4791 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
4792 if ((Val & 0xff) != Val) {
4793 HasSwiftVersionFlag = true;
4794 SwiftABIVersion = (Val & 0xff00) >> 8;
4795 SwiftMajorVersion = (Val & 0xff000000) >> 24;
4796 SwiftMinorVersion = (Val & 0xff0000) >> 16;
4797 }
4798 Metadata *Ops[3] = {
4799 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
4800 Op->getOperand(1),
4801 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
4802 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4803 Changed = true;
4804 }
4805 }
4806 }
4807
4808 // "Objective-C Class Properties" is recently added for Objective-C. We
4809 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
4810 // flag of value 0, so we can correclty downgrade this flag when trying to
4811 // link an ObjC bitcode without this module flag with an ObjC bitcode with
4812 // this module flag.
4813 if (HasObjCFlag && !HasClassProperties) {
4814 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
4815 (uint32_t)0);
4816 Changed = true;
4817 }
4818
4819 if (HasSwiftVersionFlag) {
4820 M.addModuleFlag(Module::Error, "Swift ABI Version",
4821 SwiftABIVersion);
4822 M.addModuleFlag(Module::Error, "Swift Major Version",
4823 ConstantInt::get(Int8Ty, SwiftMajorVersion));
4824 M.addModuleFlag(Module::Error, "Swift Minor Version",
4825 ConstantInt::get(Int8Ty, SwiftMinorVersion));
4826 Changed = true;
4827 }
4828
4829 return Changed;
4830}
4831
4832void llvm::UpgradeSectionAttributes(Module &M) {
4833 auto TrimSpaces = [](StringRef Section) -> std::string {
4834 SmallVector<StringRef, 5> Components;
4835 Section.split(Components, ',');
4836
4837 SmallString<32> Buffer;
4838 raw_svector_ostream OS(Buffer);
4839
4840 for (auto Component : Components)
4841 OS << ',' << Component.trim();
4842
4843 return std::string(OS.str().substr(1));
4844 };
4845
4846 for (auto &GV : M.globals()) {
4847 if (!GV.hasSection())
4848 continue;
4849
4850 StringRef Section = GV.getSection();
4851
4852 if (!Section.startswith("__DATA, __objc_catlist"))
4853 continue;
4854
4855 // __DATA, __objc_catlist, regular, no_dead_strip
4856 // __DATA,__objc_catlist,regular,no_dead_strip
4857 GV.setSection(TrimSpaces(Section));
4858 }
4859}
4860
4861namespace {
4862// Prior to LLVM 10.0, the strictfp attribute could be used on individual
4863// callsites within a function that did not also have the strictfp attribute.
4864// Since 10.0, if strict FP semantics are needed within a function, the
4865// function must have the strictfp attribute and all calls within the function
4866// must also have the strictfp attribute. This latter restriction is
4867// necessary to prevent unwanted libcall simplification when a function is
4868// being cloned (such as for inlining).
4869//
4870// The "dangling" strictfp attribute usage was only used to prevent constant
4871// folding and other libcall simplification. The nobuiltin attribute on the
4872// callsite has the same effect.
4873struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
4874 StrictFPUpgradeVisitor() = default;
4875
4876 void visitCallBase(CallBase &Call) {
4877 if (!Call.isStrictFP())
4878 return;
4879 if (isa<ConstrainedFPIntrinsic>(&Call))
4880 return;
4881 // If we get here, the caller doesn't have the strictfp attribute
4882 // but this callsite does. Replace the strictfp attribute with nobuiltin.
4883 Call.removeFnAttr(Attribute::StrictFP);
4884 Call.addFnAttr(Attribute::NoBuiltin);
4885 }
4886};
4887} // namespace
4888
4889void llvm::UpgradeFunctionAttributes(Function &F) {
4890 // If a function definition doesn't have the strictfp attribute,
4891 // convert any callsite strictfp attributes to nobuiltin.
4892 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
4893 StrictFPUpgradeVisitor SFPV;
4894 SFPV.visit(F);
4895 }
4896
4897 // Remove all incompatibile attributes from function.
4898 F.removeRetAttrs(AttributeFuncs::typeIncompatible(F.getReturnType()));
4899 for (auto &Arg : F.args())
4900 Arg.removeAttrs(AttributeFuncs::typeIncompatible(Arg.getType()));
4901}
4902
4903static bool isOldLoopArgument(Metadata *MD) {
4904 auto *T = dyn_cast_or_null<MDTuple>(MD);
4905 if (!T)
4906 return false;
4907 if (T->getNumOperands() < 1)
4908 return false;
4909 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
4910 if (!S)
4911 return false;
4912 return S->getString().startswith("llvm.vectorizer.");
4913}
4914
4915static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
4916 StringRef OldPrefix = "llvm.vectorizer.";
4917 assert(OldTag.startswith(OldPrefix) && "Expected old prefix")(static_cast <bool> (OldTag.startswith(OldPrefix) &&
"Expected old prefix") ? void (0) : __assert_fail ("OldTag.startswith(OldPrefix) && \"Expected old prefix\""
, "llvm/lib/IR/AutoUpgrade.cpp", 4917, __extension__ __PRETTY_FUNCTION__
));
4918
4919 if (OldTag == "llvm.vectorizer.unroll")
4920 return MDString::get(C, "llvm.loop.interleave.count");
4921
4922 return MDString::get(
4923 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
4924 .str());
4925}
4926
4927static Metadata *upgradeLoopArgument(Metadata *MD) {
4928 auto *T = dyn_cast_or_null<MDTuple>(MD);
4929 if (!T)
4930 return MD;
4931 if (T->getNumOperands() < 1)
4932 return MD;
4933 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
4934 if (!OldTag)
4935 return MD;
4936 if (!OldTag->getString().startswith("llvm.vectorizer."))
4937 return MD;
4938
4939 // This has an old tag. Upgrade it.
4940 SmallVector<Metadata *, 8> Ops;
4941 Ops.reserve(T->getNumOperands());
4942 Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString()));
4943 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
4944 Ops.push_back(T->getOperand(I));
4945
4946 return MDTuple::get(T->getContext(), Ops);
4947}
4948
4949MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
4950 auto *T = dyn_cast<MDTuple>(&N);
4951 if (!T)
4952 return &N;
4953
4954 if (none_of(T->operands(), isOldLoopArgument))
4955 return &N;
4956
4957 SmallVector<Metadata *, 8> Ops;
4958 Ops.reserve(T->getNumOperands());
4959 for (Metadata *MD : T->operands())
4960 Ops.push_back(upgradeLoopArgument(MD));
4961
4962 return MDTuple::get(T->getContext(), Ops);
4963}
4964
4965std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
4966 Triple T(TT);
4967 // The only data layout upgrades needed for pre-GCN are setting the address
4968 // space of globals to 1.
4969 if (T.isAMDGPU() && !T.isAMDGCN() && !DL.contains("-G") &&
4970 !DL.startswith("G")) {
4971 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
4972 }
4973
4974 if (T.isRISCV64()) {
4975 // Make i32 a native type for 64-bit RISC-V.
4976 auto I = DL.find("-n64-");
4977 if (I != StringRef::npos)
4978 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
4979 return DL.str();
4980 }
4981
4982 std::string Res = DL.str();
4983 // AMDGCN data layout upgrades.
4984 if (T.isAMDGCN()) {
4985 // Define address spaces for constants.
4986 if (!DL.contains("-G") && !DL.starts_with("G"))
4987 Res.append(Res.empty() ? "G1" : "-G1");
4988
4989 // Add missing non-integral declarations.
4990 // This goes before adding new address spaces to prevent incoherent string
4991 // values.
4992 if (!DL.contains("-ni") && !DL.startswith("ni"))
4993 Res.append("-ni:7:8");
4994 // Update ni:7 to ni:7:8.
4995 if (DL.ends_with("ni:7"))
4996 Res.append(":8");
4997
4998 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
4999 // resources) An empty data layout has already been upgraded to G1 by now.
5000 if (!DL.contains("-p7") && !DL.startswith("p7"))
5001 Res.append("-p7:160:256:256:32");
5002 if (!DL.contains("-p8") && !DL.startswith("p8"))
5003 Res.append("-p8:128:128");
5004
5005 return Res;
5006 }
5007
5008 if (!T.isX86())
5009 return Res;
5010
5011 // If the datalayout matches the expected format, add pointer size address
5012 // spaces to the datalayout.
5013 std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";
5014 if (!DL.contains(AddrSpaces)) {
5015 SmallVector<StringRef, 4> Groups;
5016 Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
5017 if (R.match(DL, &Groups))
5018 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
5019 }
5020
5021 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
5022 // Raising the alignment is safe because Clang did not produce f80 values in
5023 // the MSVC environment before this upgrade was added.
5024 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
5025 StringRef Ref = Res;
5026 auto I = Ref.find("-f80:32-");
5027 if (I != StringRef::npos)
5028 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
5029 }
5030
5031 return Res;
5032}
5033
5034void llvm::UpgradeAttributes(AttrBuilder &B) {
5035 StringRef FramePointer;
5036 Attribute A = B.getAttribute("no-frame-pointer-elim");
5037 if (A.isValid()) {
5038 // The value can be "true" or "false".
5039 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
5040 B.removeAttribute("no-frame-pointer-elim");
5041 }
5042 if (B.contains("no-frame-pointer-elim-non-leaf")) {
5043 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
5044 if (FramePointer != "all")
5045 FramePointer = "non-leaf";
5046 B.removeAttribute("no-frame-pointer-elim-non-leaf");
5047 }
5048 if (!FramePointer.empty())
5049 B.addAttribute("frame-pointer", FramePointer);
5050
5051 A = B.getAttribute("null-pointer-is-valid");
5052 if (A.isValid()) {
5053 // The value can be "true" or "false".
5054 bool NullPointerIsValid = A.getValueAsString() == "true";
5055 B.removeAttribute("null-pointer-is-valid");
5056 if (NullPointerIsValid)
5057 B.addAttribute(Attribute::NullPointerIsValid);
5058 }
5059}
5060
5061void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
5062 // clang.arc.attachedcall bundles are now required to have an operand.
5063 // If they don't, it's okay to drop them entirely: when there is an operand,
5064 // the "attachedcall" is meaningful and required, but without an operand,
5065 // it's just a marker NOP. Dropping it merely prevents an optimization.
5066 erase_if(Bundles, [&](OperandBundleDef &OBD) {
5067 return OBD.getTag() == "clang.arc.attachedcall" &&
5068 OBD.inputs().empty();
5069 });
5070}