Bug Summary

File:lib/IR/AutoUpgrade.cpp
Warning:line 861, column 32
Division by zero

Annotated Source Code

/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp

→
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the auto-upgrade helper functions.
11// This is where deprecated IR intrinsics and other IR features are updated to
12// current specifications.
13//
14//===----------------------------------------------------------------------===//
15
16#include "llvm/IR/AutoUpgrade.h"
17#include "llvm/ADT/StringSwitch.h"
18#include "llvm/IR/CFG.h"
19#include "llvm/IR/CallSite.h"
20#include "llvm/IR/Constants.h"
21#include "llvm/IR/DIBuilder.h"
22#include "llvm/IR/DebugInfo.h"
23#include "llvm/IR/DiagnosticInfo.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/Instruction.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/Module.h"
30#include "llvm/IR/Verifier.h"
31#include "llvm/Support/ErrorHandling.h"
32#include "llvm/Support/Regex.h"
33#include <cstring>
34using namespace llvm;
35
36static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
37
38// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
39// changed their type from v4f32 to v2i64.
40static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID,
41 Function *&NewFn) {
42 // Check whether this is an old version of the function, which received
43 // v4f32 arguments.
44 Type *Arg0Type = F->getFunctionType()->getParamType(0);
45 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
46 return false;
47
48 // Yes, it's old, replace it with new version.
49 rename(F);
50 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
51 return true;
52}
53
54// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
55// arguments have changed their type from i32 to i8.
56static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
57 Function *&NewFn) {
58 // Check that the last argument is an i32.
59 Type *LastArgType = F->getFunctionType()->getParamType(
60 F->getFunctionType()->getNumParams() - 1);
61 if (!LastArgType->isIntegerTy(32))
62 return false;
63
64 // Move this function aside and map down.
65 rename(F);
66 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
67 return true;
68}
69
70static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
71 // All of the intrinsics matches below should be marked with which llvm
72 // version started autoupgrading them. At some point in the future we would
73 // like to use this information to remove upgrade code for some older
74 // intrinsics. It is currently undecided how we will determine that future
75 // point.
76 if (Name=="ssse3.pabs.b.128" || // Added in 6.0
77 Name=="ssse3.pabs.w.128" || // Added in 6.0
78 Name=="ssse3.pabs.d.128" || // Added in 6.0
79 Name.startswith("avx2.pabs.") || // Added in 6.0
80 Name.startswith("avx512.mask.pabs.") || // Added in 6.0
81 Name.startswith("avx512.broadcastm") || // Added in 6.0
82 Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0
83 Name.startswith("sse2.pcmpeq.") || // Added in 3.1
84 Name.startswith("sse2.pcmpgt.") || // Added in 3.1
85 Name.startswith("avx2.pcmpeq.") || // Added in 3.1
86 Name.startswith("avx2.pcmpgt.") || // Added in 3.1
87 Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9
88 Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9
89 Name.startswith("avx.vperm2f128.") || // Added in 6.0
90 Name == "avx2.vperm2i128" || // Added in 6.0
91 Name == "sse.add.ss" || // Added in 4.0
92 Name == "sse2.add.sd" || // Added in 4.0
93 Name == "sse.sub.ss" || // Added in 4.0
94 Name == "sse2.sub.sd" || // Added in 4.0
95 Name == "sse.mul.ss" || // Added in 4.0
96 Name == "sse2.mul.sd" || // Added in 4.0
97 Name == "sse.div.ss" || // Added in 4.0
98 Name == "sse2.div.sd" || // Added in 4.0
99 Name == "sse41.pmaxsb" || // Added in 3.9
100 Name == "sse2.pmaxs.w" || // Added in 3.9
101 Name == "sse41.pmaxsd" || // Added in 3.9
102 Name == "sse2.pmaxu.b" || // Added in 3.9
103 Name == "sse41.pmaxuw" || // Added in 3.9
104 Name == "sse41.pmaxud" || // Added in 3.9
105 Name == "sse41.pminsb" || // Added in 3.9
106 Name == "sse2.pmins.w" || // Added in 3.9
107 Name == "sse41.pminsd" || // Added in 3.9
108 Name == "sse2.pminu.b" || // Added in 3.9
109 Name == "sse41.pminuw" || // Added in 3.9
110 Name == "sse41.pminud" || // Added in 3.9
111 Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0
112 Name.startswith("avx2.pmax") || // Added in 3.9
113 Name.startswith("avx2.pmin") || // Added in 3.9
114 Name.startswith("avx512.mask.pmax") || // Added in 4.0
115 Name.startswith("avx512.mask.pmin") || // Added in 4.0
116 Name.startswith("avx2.vbroadcast") || // Added in 3.8
117 Name.startswith("avx2.pbroadcast") || // Added in 3.8
118 Name.startswith("avx.vpermil.") || // Added in 3.1
119 Name.startswith("sse2.pshuf") || // Added in 3.9
120 Name.startswith("avx512.pbroadcast") || // Added in 3.9
121 Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9
122 Name.startswith("avx512.mask.movddup") || // Added in 3.9
123 Name.startswith("avx512.mask.movshdup") || // Added in 3.9
124 Name.startswith("avx512.mask.movsldup") || // Added in 3.9
125 Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9
126 Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9
127 Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9
128 Name.startswith("avx512.mask.shuf.p") || // Added in 4.0
129 Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9
130 Name.startswith("avx512.mask.perm.df.") || // Added in 3.9
131 Name.startswith("avx512.mask.perm.di.") || // Added in 3.9
132 Name.startswith("avx512.mask.punpckl") || // Added in 3.9
133 Name.startswith("avx512.mask.punpckh") || // Added in 3.9
134 Name.startswith("avx512.mask.unpckl.") || // Added in 3.9
135 Name.startswith("avx512.mask.unpckh.") || // Added in 3.9
136 Name.startswith("avx512.mask.pand.") || // Added in 3.9
137 Name.startswith("avx512.mask.pandn.") || // Added in 3.9
138 Name.startswith("avx512.mask.por.") || // Added in 3.9
139 Name.startswith("avx512.mask.pxor.") || // Added in 3.9
140 Name.startswith("avx512.mask.and.") || // Added in 3.9
141 Name.startswith("avx512.mask.andn.") || // Added in 3.9
142 Name.startswith("avx512.mask.or.") || // Added in 3.9
143 Name.startswith("avx512.mask.xor.") || // Added in 3.9
144 Name.startswith("avx512.mask.padd.") || // Added in 4.0
145 Name.startswith("avx512.mask.psub.") || // Added in 4.0
146 Name.startswith("avx512.mask.pmull.") || // Added in 4.0
147 Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
148 Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
149 Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0
150 Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0
151 Name.startswith("avx512.mask.packsswb.") || // Added in 5.0
152 Name.startswith("avx512.mask.packssdw.") || // Added in 5.0
153 Name.startswith("avx512.mask.packuswb.") || // Added in 5.0
154 Name.startswith("avx512.mask.packusdw.") || // Added in 5.0
155 Name.startswith("avx512.mask.cmp.b") || // Added in 5.0
156 Name.startswith("avx512.mask.cmp.d") || // Added in 5.0
157 Name.startswith("avx512.mask.cmp.q") || // Added in 5.0
158 Name.startswith("avx512.mask.cmp.w") || // Added in 5.0
159 Name.startswith("avx512.mask.ucmp.") || // Added in 5.0
160 Name == "avx512.mask.add.pd.128" || // Added in 4.0
161 Name == "avx512.mask.add.pd.256" || // Added in 4.0
162 Name == "avx512.mask.add.ps.128" || // Added in 4.0
163 Name == "avx512.mask.add.ps.256" || // Added in 4.0
164 Name == "avx512.mask.div.pd.128" || // Added in 4.0
165 Name == "avx512.mask.div.pd.256" || // Added in 4.0
166 Name == "avx512.mask.div.ps.128" || // Added in 4.0
167 Name == "avx512.mask.div.ps.256" || // Added in 4.0
168 Name == "avx512.mask.mul.pd.128" || // Added in 4.0
169 Name == "avx512.mask.mul.pd.256" || // Added in 4.0
170 Name == "avx512.mask.mul.ps.128" || // Added in 4.0
171 Name == "avx512.mask.mul.ps.256" || // Added in 4.0
172 Name == "avx512.mask.sub.pd.128" || // Added in 4.0
173 Name == "avx512.mask.sub.pd.256" || // Added in 4.0
174 Name == "avx512.mask.sub.ps.128" || // Added in 4.0
175 Name == "avx512.mask.sub.ps.256" || // Added in 4.0
176 Name == "avx512.mask.max.pd.128" || // Added in 5.0
177 Name == "avx512.mask.max.pd.256" || // Added in 5.0
178 Name == "avx512.mask.max.ps.128" || // Added in 5.0
179 Name == "avx512.mask.max.ps.256" || // Added in 5.0
180 Name == "avx512.mask.min.pd.128" || // Added in 5.0
181 Name == "avx512.mask.min.pd.256" || // Added in 5.0
182 Name == "avx512.mask.min.ps.128" || // Added in 5.0
183 Name == "avx512.mask.min.ps.256" || // Added in 5.0
184 Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0
185 Name.startswith("avx512.mask.psll.d") || // Added in 4.0
186 Name.startswith("avx512.mask.psll.q") || // Added in 4.0
187 Name.startswith("avx512.mask.psll.w") || // Added in 4.0
188 Name.startswith("avx512.mask.psra.d") || // Added in 4.0
189 Name.startswith("avx512.mask.psra.q") || // Added in 4.0
190 Name.startswith("avx512.mask.psra.w") || // Added in 4.0
191 Name.startswith("avx512.mask.psrl.d") || // Added in 4.0
192 Name.startswith("avx512.mask.psrl.q") || // Added in 4.0
193 Name.startswith("avx512.mask.psrl.w") || // Added in 4.0
194 Name.startswith("avx512.mask.pslli") || // Added in 4.0
195 Name.startswith("avx512.mask.psrai") || // Added in 4.0
196 Name.startswith("avx512.mask.psrli") || // Added in 4.0
197 Name.startswith("avx512.mask.psllv") || // Added in 4.0
198 Name.startswith("avx512.mask.psrav") || // Added in 4.0
199 Name.startswith("avx512.mask.psrlv") || // Added in 4.0
200 Name.startswith("sse41.pmovsx") || // Added in 3.8
201 Name.startswith("sse41.pmovzx") || // Added in 3.9
202 Name.startswith("avx2.pmovsx") || // Added in 3.9
203 Name.startswith("avx2.pmovzx") || // Added in 3.9
204 Name.startswith("avx512.mask.pmovsx") || // Added in 4.0
205 Name.startswith("avx512.mask.pmovzx") || // Added in 4.0
206 Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0
207 Name == "sse2.cvtdq2pd" || // Added in 3.9
208 Name == "sse2.cvtps2pd" || // Added in 3.9
209 Name == "avx.cvtdq2.pd.256" || // Added in 3.9
210 Name == "avx.cvt.ps2.pd.256" || // Added in 3.9
211 Name.startswith("avx.vinsertf128.") || // Added in 3.7
212 Name == "avx2.vinserti128" || // Added in 3.7
213 Name.startswith("avx512.mask.insert") || // Added in 4.0
214 Name.startswith("avx.vextractf128.") || // Added in 3.7
215 Name == "avx2.vextracti128" || // Added in 3.7
216 Name.startswith("avx512.mask.vextract") || // Added in 4.0
217 Name.startswith("sse4a.movnt.") || // Added in 3.9
218 Name.startswith("avx.movnt.") || // Added in 3.2
219 Name.startswith("avx512.storent.") || // Added in 3.9
220 Name == "sse41.movntdqa" || // Added in 5.0
221 Name == "avx2.movntdqa" || // Added in 5.0
222 Name == "avx512.movntdqa" || // Added in 5.0
223 Name == "sse2.storel.dq" || // Added in 3.9
224 Name.startswith("sse.storeu.") || // Added in 3.9
225 Name.startswith("sse2.storeu.") || // Added in 3.9
226 Name.startswith("avx.storeu.") || // Added in 3.9
227 Name.startswith("avx512.mask.storeu.") || // Added in 3.9
228 Name.startswith("avx512.mask.store.p") || // Added in 3.9
229 Name.startswith("avx512.mask.store.b.") || // Added in 3.9
230 Name.startswith("avx512.mask.store.w.") || // Added in 3.9
231 Name.startswith("avx512.mask.store.d.") || // Added in 3.9
232 Name.startswith("avx512.mask.store.q.") || // Added in 3.9
233 Name.startswith("avx512.mask.loadu.") || // Added in 3.9
234 Name.startswith("avx512.mask.load.") || // Added in 3.9
235 Name == "sse42.crc32.64.8" || // Added in 3.4
236 Name.startswith("avx.vbroadcast.s") || // Added in 3.5
237 Name.startswith("avx512.mask.palignr.") || // Added in 3.9
238 Name.startswith("avx512.mask.valign.") || // Added in 4.0
239 Name.startswith("sse2.psll.dq") || // Added in 3.7
240 Name.startswith("sse2.psrl.dq") || // Added in 3.7
241 Name.startswith("avx2.psll.dq") || // Added in 3.7
242 Name.startswith("avx2.psrl.dq") || // Added in 3.7
243 Name.startswith("avx512.psll.dq") || // Added in 3.9
244 Name.startswith("avx512.psrl.dq") || // Added in 3.9
245 Name == "sse41.pblendw" || // Added in 3.7
246 Name.startswith("sse41.blendp") || // Added in 3.7
247 Name.startswith("avx.blend.p") || // Added in 3.7
248 Name == "avx2.pblendw" || // Added in 3.7
249 Name.startswith("avx2.pblendd.") || // Added in 3.7
250 Name.startswith("avx.vbroadcastf128") || // Added in 4.0
251 Name == "avx2.vbroadcasti128" || // Added in 3.7
252 Name.startswith("avx512.mask.broadcastf") || // Added in 6.0
253 Name.startswith("avx512.mask.broadcasti") || // Added in 6.0
254 Name == "xop.vpcmov" || // Added in 3.8
255 Name == "xop.vpcmov.256" || // Added in 5.0
256 Name.startswith("avx512.mask.move.s") || // Added in 4.0
257 Name.startswith("avx512.cvtmask2") || // Added in 5.0
258 (Name.startswith("xop.vpcom") && // Added in 3.2
259 F->arg_size() == 2) ||
260 Name.startswith("sse2.pavg") || // Added in 6.0
261 Name.startswith("avx2.pavg") || // Added in 6.0
262 Name.startswith("avx512.mask.pavg")) // Added in 6.0
263 return true;
264
265 return false;
266}
267
268static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
269 Function *&NewFn) {
270 // Only handle intrinsics that start with "x86.".
271 if (!Name.startswith("x86."))
272 return false;
273 // Remove "x86." prefix.
274 Name = Name.substr(4);
275
276 if (ShouldUpgradeX86Intrinsic(F, Name)) {
277 NewFn = nullptr;
278 return true;
279 }
280
281 // SSE4.1 ptest functions may have an old signature.
282 if (Name.startswith("sse41.ptest")) { // Added in 3.2
283 if (Name.substr(11) == "c")
284 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn);
285 if (Name.substr(11) == "z")
286 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn);
287 if (Name.substr(11) == "nzc")
288 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
289 }
290 // Several blend and other instructions with masks used the wrong number of
291 // bits.
292 if (Name == "sse41.insertps") // Added in 3.6
293 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
294 NewFn);
295 if (Name == "sse41.dppd") // Added in 3.6
296 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
297 NewFn);
298 if (Name == "sse41.dpps") // Added in 3.6
299 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
300 NewFn);
301 if (Name == "sse41.mpsadbw") // Added in 3.6
302 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
303 NewFn);
304 if (Name == "avx.dp.ps.256") // Added in 3.6
305 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
306 NewFn);
307 if (Name == "avx2.mpsadbw") // Added in 3.6
308 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
309 NewFn);
310
311 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
312 if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
313 rename(F);
314 NewFn = Intrinsic::getDeclaration(F->getParent(),
315 Intrinsic::x86_xop_vfrcz_ss);
316 return true;
317 }
318 if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {
319 rename(F);
320 NewFn = Intrinsic::getDeclaration(F->getParent(),
321 Intrinsic::x86_xop_vfrcz_sd);
322 return true;
323 }
324 // Upgrade any XOP PERMIL2 index operand still using a float/double vector.
325 if (Name.startswith("xop.vpermil2")) { // Added in 3.9
326 auto Idx = F->getFunctionType()->getParamType(2);
327 if (Idx->isFPOrFPVectorTy()) {
328 rename(F);
329 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
330 unsigned EltSize = Idx->getScalarSizeInBits();
331 Intrinsic::ID Permil2ID;
332 if (EltSize == 64 && IdxSize == 128)
333 Permil2ID = Intrinsic::x86_xop_vpermil2pd;
334 else if (EltSize == 32 && IdxSize == 128)
335 Permil2ID = Intrinsic::x86_xop_vpermil2ps;
336 else if (EltSize == 64 && IdxSize == 256)
337 Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
338 else
339 Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
340 NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);
341 return true;
342 }
343 }
344
345 return false;
346}
347
348static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
349 assert(F && "Illegal to upgrade a non-existent Function.")((F && "Illegal to upgrade a non-existent Function.")
? static_cast<void> (0) : __assert_fail ("F && \"Illegal to upgrade a non-existent Function.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 349, __PRETTY_FUNCTION__));
350
351 // Quickly eliminate it, if it's not a candidate.
352 StringRef Name = F->getName();
353 if (Name.size() <= 8 || !Name.startswith("llvm."))
354 return false;
355 Name = Name.substr(5); // Strip off "llvm."
356
357 switch (Name[0]) {
358 default: break;
359 case 'a': {
360 if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) {
361 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
362 F->arg_begin()->getType());
363 return true;
364 }
365 if (Name.startswith("arm.neon.vclz")) {
366 Type* args[2] = {
367 F->arg_begin()->getType(),
368 Type::getInt1Ty(F->getContext())
369 };
370 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
371 // the end of the name. Change name from llvm.arm.neon.vclz.* to
372 // llvm.ctlz.*
373 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
374 NewFn = Function::Create(fType, F->getLinkage(),
375 "llvm.ctlz." + Name.substr(14), F->getParent());
376 return true;
377 }
378 if (Name.startswith("arm.neon.vcnt")) {
379 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
380 F->arg_begin()->getType());
381 return true;
382 }
383 Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");
384 if (vldRegex.match(Name)) {
385 auto fArgs = F->getFunctionType()->params();
386 SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());
387 // Can't use Intrinsic::getDeclaration here as the return types might
388 // then only be structurally equal.
389 FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false);
390 NewFn = Function::Create(fType, F->getLinkage(),
391 "llvm." + Name + ".p0i8", F->getParent());
392 return true;
393 }
394 Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
395 if (vstRegex.match(Name)) {
396 static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
397 Intrinsic::arm_neon_vst2,
398 Intrinsic::arm_neon_vst3,
399 Intrinsic::arm_neon_vst4};
400
401 static const Intrinsic::ID StoreLaneInts[] = {
402 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
403 Intrinsic::arm_neon_vst4lane
404 };
405
406 auto fArgs = F->getFunctionType()->params();
407 Type *Tys[] = {fArgs[0], fArgs[1]};
408 if (Name.find("lane") == StringRef::npos)
409 NewFn = Intrinsic::getDeclaration(F->getParent(),
410 StoreInts[fArgs.size() - 3], Tys);
411 else
412 NewFn = Intrinsic::getDeclaration(F->getParent(),
413 StoreLaneInts[fArgs.size() - 5], Tys);
414 return true;
415 }
416 if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
417 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
418 return true;
419 }
420 break;
421 }
422
423 case 'c': {
424 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
425 rename(F);
426 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
427 F->arg_begin()->getType());
428 return true;
429 }
430 if (Name.startswith("cttz.") && F->arg_size() == 1) {
431 rename(F);
432 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
433 F->arg_begin()->getType());
434 return true;
435 }
436 break;
437 }
438 case 'd': {
439 if (Name == "dbg.value" && F->arg_size() == 4) {
440 rename(F);
441 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
442 return true;
443 }
444 break;
445 }
446 case 'i':
447 case 'l': {
448 bool IsLifetimeStart = Name.startswith("lifetime.start");
449 if (IsLifetimeStart || Name.startswith("invariant.start")) {
450 Intrinsic::ID ID = IsLifetimeStart ?
451 Intrinsic::lifetime_start : Intrinsic::invariant_start;
452 auto Args = F->getFunctionType()->params();
453 Type* ObjectPtr[1] = {Args[1]};
454 if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
455 rename(F);
456 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
457 return true;
458 }
459 }
460
461 bool IsLifetimeEnd = Name.startswith("lifetime.end");
462 if (IsLifetimeEnd || Name.startswith("invariant.end")) {
463 Intrinsic::ID ID = IsLifetimeEnd ?
464 Intrinsic::lifetime_end : Intrinsic::invariant_end;
465
466 auto Args = F->getFunctionType()->params();
467 Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]};
468 if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
469 rename(F);
470 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
471 return true;
472 }
473 }
474 break;
475 }
476 case 'm': {
477 if (Name.startswith("masked.load.")) {
478 Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };
479 if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) {
480 rename(F);
481 NewFn = Intrinsic::getDeclaration(F->getParent(),
482 Intrinsic::masked_load,
483 Tys);
484 return true;
485 }
486 }
487 if (Name.startswith("masked.store.")) {
488 auto Args = F->getFunctionType()->params();
489 Type *Tys[] = { Args[0], Args[1] };
490 if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) {
491 rename(F);
492 NewFn = Intrinsic::getDeclaration(F->getParent(),
493 Intrinsic::masked_store,
494 Tys);
495 return true;
496 }
497 }
498 // Renaming gather/scatter intrinsics with no address space overloading
499 // to the new overload which includes an address space
500 if (Name.startswith("masked.gather.")) {
501 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
502 if (F->getName() != Intrinsic::getName(Intrinsic::masked_gather, Tys)) {
503 rename(F);
504 NewFn = Intrinsic::getDeclaration(F->getParent(),
505 Intrinsic::masked_gather, Tys);
506 return true;
507 }
508 }
509 if (Name.startswith("masked.scatter.")) {
510 auto Args = F->getFunctionType()->params();
511 Type *Tys[] = {Args[0], Args[1]};
512 if (F->getName() != Intrinsic::getName(Intrinsic::masked_scatter, Tys)) {
513 rename(F);
514 NewFn = Intrinsic::getDeclaration(F->getParent(),
515 Intrinsic::masked_scatter, Tys);
516 return true;
517 }
518 }
519 break;
520 }
521 case 'n': {
522 if (Name.startswith("nvvm.")) {
523 Name = Name.substr(5);
524
525 // The following nvvm intrinsics correspond exactly to an LLVM intrinsic.
526 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name)
527 .Cases("brev32", "brev64", Intrinsic::bitreverse)
528 .Case("clz.i", Intrinsic::ctlz)
529 .Case("popc.i", Intrinsic::ctpop)
530 .Default(Intrinsic::not_intrinsic);
531 if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) {
532 NewFn = Intrinsic::getDeclaration(F->getParent(), IID,
533 {F->getReturnType()});
534 return true;
535 }
536
537 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
538 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
539 //
540 // TODO: We could add lohi.i2d.
541 bool Expand = StringSwitch<bool>(Name)
542 .Cases("abs.i", "abs.ll", true)
543 .Cases("clz.ll", "popc.ll", "h2f", true)
544 .Cases("max.i", "max.ll", "max.ui", "max.ull", true)
545 .Cases("min.i", "min.ll", "min.ui", "min.ull", true)
546 .Default(false);
547 if (Expand) {
548 NewFn = nullptr;
549 return true;
550 }
551 }
552 break;
553 }
554 case 'o':
555 // We only need to change the name to match the mangling including the
556 // address space.
557 if (Name.startswith("objectsize.")) {
558 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
559 if (F->arg_size() == 2 ||
560 F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
561 rename(F);
562 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
563 Tys);
564 return true;
565 }
566 }
567 break;
568
569 case 's':
570 if (Name == "stackprotectorcheck") {
571 NewFn = nullptr;
572 return true;
573 }
574 break;
575
576 case 'x':
577 if (UpgradeX86IntrinsicFunction(F, Name, NewFn))
578 return true;
579 }
580 // Remangle our intrinsic since we upgrade the mangling
581 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
582 if (Result != None) {
583 NewFn = Result.getValue();
584 return true;
585 }
586
587 // This may not belong here. This function is effectively being overloaded
588 // to both detect an intrinsic which needs upgrading, and to provide the
589 // upgraded form of the intrinsic. We should perhaps have two separate
590 // functions for this.
591 return false;
592}
593
594bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
595 NewFn = nullptr;
596 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
597 assert(F != NewFn && "Intrinsic function upgraded to the same function")((F != NewFn && "Intrinsic function upgraded to the same function"
) ? static_cast<void> (0) : __assert_fail ("F != NewFn && \"Intrinsic function upgraded to the same function\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 597, __PRETTY_FUNCTION__));
598
599 // Upgrade intrinsic attributes. This does not change the function.
600 if (NewFn)
601 F = NewFn;
602 if (Intrinsic::ID id = F->getIntrinsicID())
603 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
604 return Upgraded;
605}
606
607bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
608 // Nothing to do yet.
609 return false;
610}
611
612// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
613// to byte shuffles.
614static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
615 Value *Op, unsigned Shift) {
616 Type *ResultTy = Op->getType();
617 unsigned NumElts = ResultTy->getVectorNumElements() * 8;
618
619 // Bitcast from a 64-bit element type to a byte element type.
620 Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts);
621 Op = Builder.CreateBitCast(Op, VecTy, "cast");
622
623 // We'll be shuffling in zeroes.
624 Value *Res = Constant::getNullValue(VecTy);
625
626 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
627 // we'll just return the zero vector.
628 if (Shift < 16) {
629 uint32_t Idxs[64];
630 // 256/512-bit version is split into 2/4 16-byte lanes.
631 for (unsigned l = 0; l != NumElts; l += 16)
632 for (unsigned i = 0; i != 16; ++i) {
633 unsigned Idx = NumElts + i - Shift;
634 if (Idx < NumElts)
635 Idx -= NumElts - 16; // end of lane, switch operand.
636 Idxs[l + i] = Idx + l;
637 }
638
639 Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));
640 }
641
642 // Bitcast back to a 64-bit element type.
643 return Builder.CreateBitCast(Res, ResultTy, "cast");
644}
645
646// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
647// to byte shuffles.
648static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
649 unsigned Shift) {
650 Type *ResultTy = Op->getType();
651 unsigned NumElts = ResultTy->getVectorNumElements() * 8;
652
653 // Bitcast from a 64-bit element type to a byte element type.
654 Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts);
655 Op = Builder.CreateBitCast(Op, VecTy, "cast");
656
657 // We'll be shuffling in zeroes.
658 Value *Res = Constant::getNullValue(VecTy);
659
660 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
661 // we'll just return the zero vector.
662 if (Shift < 16) {
663 uint32_t Idxs[64];
664 // 256/512-bit version is split into 2/4 16-byte lanes.
665 for (unsigned l = 0; l != NumElts; l += 16)
666 for (unsigned i = 0; i != 16; ++i) {
667 unsigned Idx = i + Shift;
668 if (Idx >= 16)
669 Idx += NumElts - 16; // end of lane, switch operand.
670 Idxs[l + i] = Idx + l;
671 }
672
673 Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));
674 }
675
676 // Bitcast back to a 64-bit element type.
677 return Builder.CreateBitCast(Res, ResultTy, "cast");
678}
679
680static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
681 unsigned NumElts) {
682 llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(),
683 cast<IntegerType>(Mask->getType())->getBitWidth());
684 Mask = Builder.CreateBitCast(Mask, MaskTy);
685
686 // If we have less than 8 elements, then the starting mask was an i8 and
687 // we need to extract down to the right number of elements.
688 if (NumElts < 8) {
689 uint32_t Indices[4];
690 for (unsigned i = 0; i != NumElts; ++i)
691 Indices[i] = i;
692 Mask = Builder.CreateShuffleVector(Mask, Mask,
693 makeArrayRef(Indices, NumElts),
694 "extract");
695 }
696
697 return Mask;
698}
699
700static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
701 Value *Op0, Value *Op1) {
702 // If the mask is all ones just emit the align operation.
703 if (const auto *C = dyn_cast<Constant>(Mask))
704 if (C->isAllOnesValue())
705 return Op0;
706
707 Mask = getX86MaskVec(Builder, Mask, Op0->getType()->getVectorNumElements());
708 return Builder.CreateSelect(Mask, Op0, Op1);
709}
710
711// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
712// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
713// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
714static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
715 Value *Op1, Value *Shift,
716 Value *Passthru, Value *Mask,
717 bool IsVALIGN) {
718 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
719
720 unsigned NumElts = Op0->getType()->getVectorNumElements();
721 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!")(((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!"
) ? static_cast<void> (0) : __assert_fail ("(IsVALIGN || NumElts % 16 == 0) && \"Illegal NumElts for PALIGNR!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 721, __PRETTY_FUNCTION__));
722 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!")(((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!"
) ? static_cast<void> (0) : __assert_fail ("(!IsVALIGN || NumElts <= 16) && \"NumElts too large for VALIGN!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 722, __PRETTY_FUNCTION__));
723 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!")((isPowerOf2_32(NumElts) && "NumElts not a power of 2!"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(NumElts) && \"NumElts not a power of 2!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 723, __PRETTY_FUNCTION__));
724
725 // Mask the immediate for VALIGN.
726 if (IsVALIGN)
727 ShiftVal &= (NumElts - 1);
728
729 // If palignr is shifting the pair of vectors more than the size of two
730 // lanes, emit zero.
731 if (ShiftVal >= 32)
732 return llvm::Constant::getNullValue(Op0->getType());
733
734 // If palignr is shifting the pair of input vectors more than one lane,
735 // but less than two lanes, convert to shifting in zeroes.
736 if (ShiftVal > 16) {
737 ShiftVal -= 16;
738 Op1 = Op0;
739 Op0 = llvm::Constant::getNullValue(Op0->getType());
740 }
741
742 uint32_t Indices[64];
743 // 256-bit palignr operates on 128-bit lanes so we need to handle that
744 for (unsigned l = 0; l < NumElts; l += 16) {
745 for (unsigned i = 0; i != 16; ++i) {
746 unsigned Idx = ShiftVal + i;
747 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
748 Idx += NumElts - 16; // End of lane, switch operand.
749 Indices[l + i] = Idx + l;
750 }
751 }
752
753 Value *Align = Builder.CreateShuffleVector(Op1, Op0,
754 makeArrayRef(Indices, NumElts),
755 "palignr");
756
757 return EmitX86Select(Builder, Mask, Align, Passthru);
758}
759
760static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
761 Value *Ptr, Value *Data, Value *Mask,
762 bool Aligned) {
763 // Cast the pointer to the right type.
764 Ptr = Builder.CreateBitCast(Ptr,
765 llvm::PointerType::getUnqual(Data->getType()));
766 unsigned Align =
767 Aligned ? cast<VectorType>(Data->getType())->getBitWidth() / 8 : 1;
768
769 // If the mask is all ones just emit a regular store.
770 if (const auto *C = dyn_cast<Constant>(Mask))
771 if (C->isAllOnesValue())
772 return Builder.CreateAlignedStore(Data, Ptr, Align);
773
774 // Convert the mask from an integer type to a vector of i1.
775 unsigned NumElts = Data->getType()->getVectorNumElements();
776 Mask = getX86MaskVec(Builder, Mask, NumElts);
777 return Builder.CreateMaskedStore(Data, Ptr, Align, Mask);
778}
779
780static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
781 Value *Ptr, Value *Passthru, Value *Mask,
782 bool Aligned) {
783 // Cast the pointer to the right type.
784 Ptr = Builder.CreateBitCast(Ptr,
785 llvm::PointerType::getUnqual(Passthru->getType()));
786 unsigned Align =
787 Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1;
788
789 // If the mask is all ones just emit a regular store.
790 if (const auto *C = dyn_cast<Constant>(Mask))
791 if (C->isAllOnesValue())
792 return Builder.CreateAlignedLoad(Ptr, Align);
793
794 // Convert the mask from an integer type to a vector of i1.
795 unsigned NumElts = Passthru->getType()->getVectorNumElements();
796 Mask = getX86MaskVec(Builder, Mask, NumElts);
797 return Builder.CreateMaskedLoad(Ptr, Align, Mask, Passthru);
798}
799
800static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) {
801 Value *Op0 = CI.getArgOperand(0);
802 llvm::Type *Ty = Op0->getType();
803 Value *Zero = llvm::Constant::getNullValue(Ty);
804 Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SGT, Op0, Zero);
805 Value *Neg = Builder.CreateNeg(Op0);
806 Value *Res = Builder.CreateSelect(Cmp, Op0, Neg);
807
808 if (CI.getNumArgOperands() == 3)
809 Res = EmitX86Select(Builder,CI.getArgOperand(2), Res, CI.getArgOperand(1));
810
811 return Res;
812}
813
814static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI,
815 ICmpInst::Predicate Pred) {
816 Value *Op0 = CI.getArgOperand(0);
817 Value *Op1 = CI.getArgOperand(1);
818 Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1);
819 Value *Res = Builder.CreateSelect(Cmp, Op0, Op1);
820
821 if (CI.getNumArgOperands() == 4)
822 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
823
824 return Res;
825}
826
827static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,
828 unsigned CC, bool Signed) {
829 Value *Op0 = CI.getArgOperand(0);
830 unsigned NumElts = Op0->getType()->getVectorNumElements();
1
Calling 'Type::getVectorNumElements'
→
27
←
Returning from 'Type::getVectorNumElements'
→
28
←
'NumElts' initialized here
→
831
832 Value *Cmp;
833 if (CC == 3) {
29
←
Assuming 'CC' is not equal to 3
→
30
←
Taking false branch
→
834 Cmp = Constant::getNullValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts));
835 } else if (CC == 7) {
31
←
Assuming 'CC' is not equal to 7
→
32
←
Taking false branch
→
836 Cmp = Constant::getAllOnesValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts));
837 } else {
838 ICmpInst::Predicate Pred;
839 switch (CC) {
33
←
Control jumps to 'case 6:' at line 846
→
840 default: llvm_unreachable("Unknown condition code")::llvm::llvm_unreachable_internal("Unknown condition code", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 840);
841 case 0: Pred = ICmpInst::ICMP_EQ; break;
842 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
843 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
844 case 4: Pred = ICmpInst::ICMP_NE; break;
845 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
846 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
34
←
Assuming 'Signed' is 0
→
35
←
'?' condition is false
→
36
←
 Execution continues on line 848
→
847 }
848 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
849 }
850
851 Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1);
852 const auto *C = dyn_cast<Constant>(Mask);
853 if (!C || !C->isAllOnesValue())
37
←
Assuming 'C' is non-null
→
38
←
Assuming the condition is false
→
39
←
Taking false branch
→
854 Cmp = Builder.CreateAnd(Cmp, getX86MaskVec(Builder, Mask, NumElts));
855
856 if (NumElts < 8) {
40
←
Assuming 'NumElts' is < 8
→
41
←
Taking true branch
→
857 uint32_t Indices[8];
858 for (unsigned i = 0; i != NumElts; ++i)
42
←
Assuming 'i' is equal to 'NumElts'
→
43
←
Loop condition is false. Execution continues on line 860
→
859 Indices[i] = i;
860 for (unsigned i = NumElts; i != 8; ++i)
44
←
Loop condition is true. Entering loop body
→
861 Indices[i] = NumElts + i % NumElts;
45
←
Division by zero
862 Cmp = Builder.CreateShuffleVector(Cmp,
863 Constant::getNullValue(Cmp->getType()),
864 Indices);
865 }
866 return Builder.CreateBitCast(Cmp, IntegerType::get(CI.getContext(),
867 std::max(NumElts, 8U)));
868}
869
870// Replace a masked intrinsic with an older unmasked intrinsic.
871static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,
872 Intrinsic::ID IID) {
873 Function *F = CI.getCalledFunction();
874 Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID);
875 Value *Rep = Builder.CreateCall(Intrin,
876 { CI.getArgOperand(0), CI.getArgOperand(1) });
877 return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
878}
879
880static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {
881 Value* A = CI.getArgOperand(0);
882 Value* B = CI.getArgOperand(1);
883 Value* Src = CI.getArgOperand(2);
884 Value* Mask = CI.getArgOperand(3);
885
886 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
887 Value* Cmp = Builder.CreateIsNotNull(AndNode);
888 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
889 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
890 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
891 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
892}
893
894
895static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) {
896 Value* Op = CI.getArgOperand(0);
897 Type* ReturnOp = CI.getType();
898 unsigned NumElts = CI.getType()->getVectorNumElements();
899 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
900 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
901}
902
903/// Upgrade a call to an old intrinsic. All argument and return casting must be
904/// provided to seamlessly integrate with existing context.
905void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
906 Function *F = CI->getCalledFunction();
907 LLVMContext &C = CI->getContext();
908 IRBuilder<> Builder(C);
909 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
910
911 assert(F && "Intrinsic call is not direct?")((F && "Intrinsic call is not direct?") ? static_cast
<void> (0) : __assert_fail ("F && \"Intrinsic call is not direct?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 911, __PRETTY_FUNCTION__));
912
913 if (!NewFn) {
914 // Get the Function's name.
915 StringRef Name = F->getName();
916
917 assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'")((Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'"
) ? static_cast<void> (0) : __assert_fail ("Name.startswith(\"llvm.\") && \"Intrinsic doesn't start with 'llvm.'\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 917, __PRETTY_FUNCTION__));
918 Name = Name.substr(5);
919
920 bool IsX86 = Name.startswith("x86.");
921 if (IsX86)
922 Name = Name.substr(4);
923 bool IsNVVM = Name.startswith("nvvm.");
924 if (IsNVVM)
925 Name = Name.substr(5);
926
927 if (IsX86 && Name.startswith("sse4a.movnt.")) {
928 Module *M = F->getParent();
929 SmallVector<Metadata *, 1> Elts;
930 Elts.push_back(
931 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
932 MDNode *Node = MDNode::get(C, Elts);
933
934 Value *Arg0 = CI->getArgOperand(0);
935 Value *Arg1 = CI->getArgOperand(1);
936
937 // Nontemporal (unaligned) store of the 0'th element of the float/double
938 // vector.
939 Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();
940 PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
941 Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");
942 Value *Extract =
943 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
944
945 StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, 1);
946 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
947
948 // Remove intrinsic.
949 CI->eraseFromParent();
950 return;
951 }
952
953 if (IsX86 && (Name.startswith("avx.movnt.") ||
954 Name.startswith("avx512.storent."))) {
955 Module *M = F->getParent();
956 SmallVector<Metadata *, 1> Elts;
957 Elts.push_back(
958 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
959 MDNode *Node = MDNode::get(C, Elts);
960
961 Value *Arg0 = CI->getArgOperand(0);
962 Value *Arg1 = CI->getArgOperand(1);
963
964 // Convert the type of the pointer to a pointer to the stored type.
965 Value *BC = Builder.CreateBitCast(Arg0,
966 PointerType::getUnqual(Arg1->getType()),
967 "cast");
968 VectorType *VTy = cast<VectorType>(Arg1->getType());
969 StoreInst *SI = Builder.CreateAlignedStore(Arg1, BC,
970 VTy->getBitWidth() / 8);
971 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
972
973 // Remove intrinsic.
974 CI->eraseFromParent();
975 return;
976 }
977
978 if (IsX86 && Name == "sse2.storel.dq") {
979 Value *Arg0 = CI->getArgOperand(0);
980 Value *Arg1 = CI->getArgOperand(1);
981
982 Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2);
983 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
984 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
985 Value *BC = Builder.CreateBitCast(Arg0,
986 PointerType::getUnqual(Elt->getType()),
987 "cast");
988 Builder.CreateAlignedStore(Elt, BC, 1);
989
990 // Remove intrinsic.
991 CI->eraseFromParent();
992 return;
993 }
994
995 if (IsX86 && (Name.startswith("sse.storeu.") ||
996 Name.startswith("sse2.storeu.") ||
997 Name.startswith("avx.storeu."))) {
998 Value *Arg0 = CI->getArgOperand(0);
999 Value *Arg1 = CI->getArgOperand(1);
1000
1001 Arg0 = Builder.CreateBitCast(Arg0,
1002 PointerType::getUnqual(Arg1->getType()),
1003 "cast");
1004 Builder.CreateAlignedStore(Arg1, Arg0, 1);
1005
1006 // Remove intrinsic.
1007 CI->eraseFromParent();
1008 return;
1009 }
1010
1011 if (IsX86 && (Name.startswith("avx512.mask.store"))) {
1012 // "avx512.mask.storeu." or "avx512.mask.store."
1013 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
1014 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1015 CI->getArgOperand(2), Aligned);
1016
1017 // Remove intrinsic.
1018 CI->eraseFromParent();
1019 return;
1020 }
1021
1022 Value *Rep;
1023 // Upgrade packed integer vector compare intrinsics to compare instructions.
1024 if (IsX86 && (Name.startswith("sse2.pcmp") ||
1025 Name.startswith("avx2.pcmp"))) {
1026 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
1027 bool CmpEq = Name[9] == 'e';
1028 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
1029 CI->getArgOperand(0), CI->getArgOperand(1));
1030 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
1031 } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) {
1032 Type *ExtTy = Type::getInt32Ty(C);
1033 if (CI->getOperand(0)->getType()->isIntegerTy(8))
1034 ExtTy = Type::getInt64Ty(C);
1035 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
1036 ExtTy->getPrimitiveSizeInBits();
1037 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
1038 Rep = Builder.CreateVectorSplat(NumElts, Rep);
1039 } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))) {
1040 unsigned NumElts =
1041 CI->getArgOperand(1)->getType()->getVectorNumElements();
1042 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
1043 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1044 CI->getArgOperand(1));
1045 } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd")) {
1046 Type *I32Ty = Type::getInt32Ty(C);
1047 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
1048 ConstantInt::get(I32Ty, 0));
1049 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
1050 ConstantInt::get(I32Ty, 0));
1051 Rep = Builder.CreateInsertElement(CI->getArgOperand(0),
1052 Builder.CreateFAdd(Elt0, Elt1),
1053 ConstantInt::get(I32Ty, 0));
1054 } else if (IsX86 && (Name == "sse.sub.ss" || Name == "sse2.sub.sd")) {
1055 Type *I32Ty = Type::getInt32Ty(C);
1056 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
1057 ConstantInt::get(I32Ty, 0));
1058 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
1059 ConstantInt::get(I32Ty, 0));
1060 Rep = Builder.CreateInsertElement(CI->getArgOperand(0),
1061 Builder.CreateFSub(Elt0, Elt1),
1062 ConstantInt::get(I32Ty, 0));
1063 } else if (IsX86 && (Name == "sse.mul.ss" || Name == "sse2.mul.sd")) {
1064 Type *I32Ty = Type::getInt32Ty(C);
1065 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
1066 ConstantInt::get(I32Ty, 0));
1067 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
1068 ConstantInt::get(I32Ty, 0));
1069 Rep = Builder.CreateInsertElement(CI->getArgOperand(0),
1070 Builder.CreateFMul(Elt0, Elt1),
1071 ConstantInt::get(I32Ty, 0));
1072 } else if (IsX86 && (Name == "sse.div.ss" || Name == "sse2.div.sd")) {
1073 Type *I32Ty = Type::getInt32Ty(C);
1074 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
1075 ConstantInt::get(I32Ty, 0));
1076 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
1077 ConstantInt::get(I32Ty, 0));
1078 Rep = Builder.CreateInsertElement(CI->getArgOperand(0),
1079 Builder.CreateFDiv(Elt0, Elt1),
1080 ConstantInt::get(I32Ty, 0));
1081 } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) {
1082 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
1083 bool CmpEq = Name[16] == 'e';
1084 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
1085 } else if (IsX86 && Name.startswith("avx512.mask.cmp")) {
1086 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
1087 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
1088 } else if (IsX86 && Name.startswith("avx512.mask.ucmp")) {
1089 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
1090 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
1091 } else if(IsX86 && (Name == "ssse3.pabs.b.128" ||
1092 Name == "ssse3.pabs.w.128" ||
1093 Name == "ssse3.pabs.d.128" ||
1094 Name.startswith("avx2.pabs") ||
1095 Name.startswith("avx512.mask.pabs"))) {
1096 Rep = upgradeAbs(Builder, *CI);
1097 } else if (IsX86 && (Name == "sse41.pmaxsb" ||
1098 Name == "sse2.pmaxs.w" ||
1099 Name == "sse41.pmaxsd" ||
1100 Name.startswith("avx2.pmaxs") ||
1101 Name.startswith("avx512.mask.pmaxs"))) {
1102 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT);
1103 } else if (IsX86 && (Name == "sse2.pmaxu.b" ||
1104 Name == "sse41.pmaxuw" ||
1105 Name == "sse41.pmaxud" ||
1106 Name.startswith("avx2.pmaxu") ||
1107 Name.startswith("avx512.mask.pmaxu"))) {
1108 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT);
1109 } else if (IsX86 && (Name == "sse41.pminsb" ||
1110 Name == "sse2.pmins.w" ||
1111 Name == "sse41.pminsd" ||
1112 Name.startswith("avx2.pmins") ||
1113 Name.startswith("avx512.mask.pmins"))) {
1114 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT);
1115 } else if (IsX86 && (Name == "sse2.pminu.b" ||
1116 Name == "sse41.pminuw" ||
1117 Name == "sse41.pminud" ||
1118 Name.startswith("avx2.pminu") ||
1119 Name.startswith("avx512.mask.pminu"))) {
1120 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT);
1121 } else if (IsX86 && (Name == "sse2.cvtdq2pd" ||
1122 Name == "sse2.cvtps2pd" ||
1123 Name == "avx.cvtdq2.pd.256" ||
1124 Name == "avx.cvt.ps2.pd.256" ||
1125 Name.startswith("avx512.mask.cvtdq2pd.") ||
1126 Name.startswith("avx512.mask.cvtudq2pd."))) {
1127 // Lossless i32/float to double conversion.
1128 // Extract the bottom elements if necessary and convert to double vector.
1129 Value *Src = CI->getArgOperand(0);
1130 VectorType *SrcTy = cast<VectorType>(Src->getType());
1131 VectorType *DstTy = cast<VectorType>(CI->getType());
1132 Rep = CI->getArgOperand(0);
1133
1134 unsigned NumDstElts = DstTy->getNumElements();
1135 if (NumDstElts < SrcTy->getNumElements()) {
1136 assert(NumDstElts == 2 && "Unexpected vector size")((NumDstElts == 2 && "Unexpected vector size") ? static_cast
<void> (0) : __assert_fail ("NumDstElts == 2 && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1136, __PRETTY_FUNCTION__));
1137 uint32_t ShuffleMask[2] = { 0, 1 };
1138 Rep = Builder.CreateShuffleVector(Rep, UndefValue::get(SrcTy),
1139 ShuffleMask);
1140 }
1141
1142 bool SInt2Double = (StringRef::npos != Name.find("cvtdq2"));
1143 bool UInt2Double = (StringRef::npos != Name.find("cvtudq2"));
1144 if (SInt2Double)
1145 Rep = Builder.CreateSIToFP(Rep, DstTy, "cvtdq2pd");
1146 else if (UInt2Double)
1147 Rep = Builder.CreateUIToFP(Rep, DstTy, "cvtudq2pd");
1148 else
1149 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
1150
1151 if (CI->getNumArgOperands() == 3)
1152 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1153 CI->getArgOperand(1));
1154 } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {
1155 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
1156 CI->getArgOperand(1), CI->getArgOperand(2),
1157 /*Aligned*/false);
1158 } else if (IsX86 && (Name.startswith("avx512.mask.load."))) {
1159 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
1160 CI->getArgOperand(1),CI->getArgOperand(2),
1161 /*Aligned*/true);
1162 } else if (IsX86 && Name.startswith("xop.vpcom")) {
1163 Intrinsic::ID intID;
1164 if (Name.endswith("ub"))
1165 intID = Intrinsic::x86_xop_vpcomub;
1166 else if (Name.endswith("uw"))
1167 intID = Intrinsic::x86_xop_vpcomuw;
1168 else if (Name.endswith("ud"))
1169 intID = Intrinsic::x86_xop_vpcomud;
1170 else if (Name.endswith("uq"))
1171 intID = Intrinsic::x86_xop_vpcomuq;
1172 else if (Name.endswith("b"))
1173 intID = Intrinsic::x86_xop_vpcomb;
1174 else if (Name.endswith("w"))
1175 intID = Intrinsic::x86_xop_vpcomw;
1176 else if (Name.endswith("d"))
1177 intID = Intrinsic::x86_xop_vpcomd;
1178 else if (Name.endswith("q"))
1179 intID = Intrinsic::x86_xop_vpcomq;
1180 else
1181 llvm_unreachable("Unknown suffix")::llvm::llvm_unreachable_internal("Unknown suffix", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1181);
1182
1183 Name = Name.substr(9); // strip off "xop.vpcom"
1184 unsigned Imm;
1185 if (Name.startswith("lt"))
1186 Imm = 0;
1187 else if (Name.startswith("le"))
1188 Imm = 1;
1189 else if (Name.startswith("gt"))
1190 Imm = 2;
1191 else if (Name.startswith("ge"))
1192 Imm = 3;
1193 else if (Name.startswith("eq"))
1194 Imm = 4;
1195 else if (Name.startswith("ne"))
1196 Imm = 5;
1197 else if (Name.startswith("false"))
1198 Imm = 6;
1199 else if (Name.startswith("true"))
1200 Imm = 7;
1201 else
1202 llvm_unreachable("Unknown condition")::llvm::llvm_unreachable_internal("Unknown condition", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1202);
1203
1204 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
1205 Rep =
1206 Builder.CreateCall(VPCOM, {CI->getArgOperand(0), CI->getArgOperand(1),
1207 Builder.getInt8(Imm)});
1208 } else if (IsX86 && Name.startswith("xop.vpcmov")) {
1209 Value *Sel = CI->getArgOperand(2);
1210 Value *NotSel = Builder.CreateNot(Sel);
1211 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
1212 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
1213 Rep = Builder.CreateOr(Sel0, Sel1);
1214 } else if (IsX86 && Name == "sse42.crc32.64.8") {
1215 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
1216 Intrinsic::x86_sse42_crc32_32_8);
1217 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
1218 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
1219 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
1220 } else if (IsX86 && Name.startswith("avx.vbroadcast.s")) {
1221 // Replace broadcasts with a series of insertelements.
1222 Type *VecTy = CI->getType();
1223 Type *EltTy = VecTy->getVectorElementType();
1224 unsigned EltNum = VecTy->getVectorNumElements();
1225 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
1226 EltTy->getPointerTo());
1227 Value *Load = Builder.CreateLoad(EltTy, Cast);
1228 Type *I32Ty = Type::getInt32Ty(C);
1229 Rep = UndefValue::get(VecTy);
1230 for (unsigned I = 0; I < EltNum; ++I)
1231 Rep = Builder.CreateInsertElement(Rep, Load,
1232 ConstantInt::get(I32Ty, I));
1233 } else if (IsX86 && (Name.startswith("sse41.pmovsx") ||
1234 Name.startswith("sse41.pmovzx") ||
1235 Name.startswith("avx2.pmovsx") ||
1236 Name.startswith("avx2.pmovzx") ||
1237 Name.startswith("avx512.mask.pmovsx") ||
1238 Name.startswith("avx512.mask.pmovzx"))) {
1239 VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType());
1240 VectorType *DstTy = cast<VectorType>(CI->getType());
1241 unsigned NumDstElts = DstTy->getNumElements();
1242
1243 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
1244 SmallVector<uint32_t, 8> ShuffleMask(NumDstElts);
1245 for (unsigned i = 0; i != NumDstElts; ++i)
1246 ShuffleMask[i] = i;
1247
1248 Value *SV = Builder.CreateShuffleVector(
1249 CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask);
1250
1251 bool DoSext = (StringRef::npos != Name.find("pmovsx"));
1252 Rep = DoSext ? Builder.CreateSExt(SV, DstTy)
1253 : Builder.CreateZExt(SV, DstTy);
1254 // If there are 3 arguments, it's a masked intrinsic so we need a select.
1255 if (CI->getNumArgOperands() == 3)
1256 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1257 CI->getArgOperand(1));
1258 } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
1259 Name == "avx2.vbroadcasti128")) {
1260 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
1261 Type *EltTy = CI->getType()->getVectorElementType();
1262 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
1263 Type *VT = VectorType::get(EltTy, NumSrcElts);
1264 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
1265 PointerType::getUnqual(VT));
1266 Value *Load = Builder.CreateAlignedLoad(Op, 1);
1267 if (NumSrcElts == 2)
1268 Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
1269 { 0, 1, 0, 1 });
1270 else
1271 Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
1272 { 0, 1, 2, 3, 0, 1, 2, 3 });
1273 } else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") ||
1274 Name.startswith("avx512.mask.broadcasti"))) {
1275 unsigned NumSrcElts =
1276 CI->getArgOperand(0)->getType()->getVectorNumElements();
1277 unsigned NumDstElts = CI->getType()->getVectorNumElements();
1278
1279 SmallVector<uint32_t, 8> ShuffleMask(NumDstElts);
1280 for (unsigned i = 0; i != NumDstElts; ++i)
1281 ShuffleMask[i] = i % NumSrcElts;
1282
1283 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
1284 CI->getArgOperand(0),
1285 ShuffleMask);
1286 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1287 CI->getArgOperand(1));
1288 } else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||
1289 Name.startswith("avx2.vbroadcast") ||
1290 Name.startswith("avx512.pbroadcast") ||
1291 Name.startswith("avx512.mask.broadcast.s"))) {
1292 // Replace vp?broadcasts with a vector shuffle.
1293 Value *Op = CI->getArgOperand(0);
1294 unsigned NumElts = CI->getType()->getVectorNumElements();
1295 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), NumElts);
1296 Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()),
1297 Constant::getNullValue(MaskTy));
1298
1299 if (CI->getNumArgOperands() == 3)
1300 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1301 CI->getArgOperand(1));
1302 } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
1303 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
1304 CI->getArgOperand(1),
1305 CI->getArgOperand(2),
1306 CI->getArgOperand(3),
1307 CI->getArgOperand(4),
1308 false);
1309 } else if (IsX86 && Name.startswith("avx512.mask.valign.")) {
1310 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
1311 CI->getArgOperand(1),
1312 CI->getArgOperand(2),
1313 CI->getArgOperand(3),
1314 CI->getArgOperand(4),
1315 true);
1316 } else if (IsX86 && (Name == "sse2.psll.dq" ||
1317 Name == "avx2.psll.dq")) {
1318 // 128/256-bit shift left specified in bits.
1319 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
1320 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
1321 Shift / 8); // Shift is in bits.
1322 } else if (IsX86 && (Name == "sse2.psrl.dq" ||
1323 Name == "avx2.psrl.dq")) {
1324 // 128/256-bit shift right specified in bits.
1325 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
1326 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
1327 Shift / 8); // Shift is in bits.
1328 } else if (IsX86 && (Name == "sse2.psll.dq.bs" ||
1329 Name == "avx2.psll.dq.bs" ||
1330 Name == "avx512.psll.dq.512")) {
1331 // 128/256/512-bit shift left specified in bytes.
1332 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
1333 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
1334 } else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||
1335 Name == "avx2.psrl.dq.bs" ||
1336 Name == "avx512.psrl.dq.512")) {
1337 // 128/256/512-bit shift right specified in bytes.
1338 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
1339 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
1340 } else if (IsX86 && (Name == "sse41.pblendw" ||
1341 Name.startswith("sse41.blendp") ||
1342 Name.startswith("avx.blend.p") ||
1343 Name == "avx2.pblendw" ||
1344 Name.startswith("avx2.pblendd."))) {
1345 Value *Op0 = CI->getArgOperand(0);
1346 Value *Op1 = CI->getArgOperand(1);
1347 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
1348 VectorType *VecTy = cast<VectorType>(CI->getType());
1349 unsigned NumElts = VecTy->getNumElements();
1350
1351 SmallVector<uint32_t, 16> Idxs(NumElts);
1352 for (unsigned i = 0; i != NumElts; ++i)
1353 Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
1354
1355 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
1356 } else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||
1357 Name == "avx2.vinserti128" ||
1358 Name.startswith("avx512.mask.insert"))) {
1359 Value *Op0 = CI->getArgOperand(0);
1360 Value *Op1 = CI->getArgOperand(1);
1361 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
1362 unsigned DstNumElts = CI->getType()->getVectorNumElements();
1363 unsigned SrcNumElts = Op1->getType()->getVectorNumElements();
1364 unsigned Scale = DstNumElts / SrcNumElts;
1365
1366 // Mask off the high bits of the immediate value; hardware ignores those.
1367 Imm = Imm % Scale;
1368
1369 // Extend the second operand into a vector the size of the destination.
1370 Value *UndefV = UndefValue::get(Op1->getType());
1371 SmallVector<uint32_t, 8> Idxs(DstNumElts);
1372 for (unsigned i = 0; i != SrcNumElts; ++i)
1373 Idxs[i] = i;
1374 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
1375 Idxs[i] = SrcNumElts;
1376 Rep = Builder.CreateShuffleVector(Op1, UndefV, Idxs);
1377
1378 // Insert the second operand into the first operand.
1379
1380 // Note that there is no guarantee that instruction lowering will actually
1381 // produce a vinsertf128 instruction for the created shuffles. In
1382 // particular, the 0 immediate case involves no lane changes, so it can
1383 // be handled as a blend.
1384
1385 // Example of shuffle mask for 32-bit elements:
1386 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
1387 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
1388
1389 // First fill with identify mask.
1390 for (unsigned i = 0; i != DstNumElts; ++i)
1391 Idxs[i] = i;
1392 // Then replace the elements where we need to insert.
1393 for (unsigned i = 0; i != SrcNumElts; ++i)
1394 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
1395 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
1396
1397 // If the intrinsic has a mask operand, handle that.
1398 if (CI->getNumArgOperands() == 5)
1399 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
1400 CI->getArgOperand(3));
1401 } else if (IsX86 && (Name.startswith("avx.vextractf128.") ||
1402 Name == "avx2.vextracti128" ||
1403 Name.startswith("avx512.mask.vextract"))) {
1404 Value *Op0 = CI->getArgOperand(0);
1405 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
1406 unsigned DstNumElts = CI->getType()->getVectorNumElements();
1407 unsigned SrcNumElts = Op0->getType()->getVectorNumElements();
1408 unsigned Scale = SrcNumElts / DstNumElts;
1409
1410 // Mask off the high bits of the immediate value; hardware ignores those.
1411 Imm = Imm % Scale;
1412
1413 // Get indexes for the subvector of the input vector.
1414 SmallVector<uint32_t, 8> Idxs(DstNumElts);
1415 for (unsigned i = 0; i != DstNumElts; ++i) {
1416 Idxs[i] = i + (Imm * DstNumElts);
1417 }
1418 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
1419
1420 // If the intrinsic has a mask operand, handle that.
1421 if (CI->getNumArgOperands() == 4)
1422 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1423 CI->getArgOperand(2));
1424 } else if (!IsX86 && Name == "stackprotectorcheck") {
1425 Rep = nullptr;
1426 } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||
1427 Name.startswith("avx512.mask.perm.di."))) {
1428 Value *Op0 = CI->getArgOperand(0);
1429 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
1430 VectorType *VecTy = cast<VectorType>(CI->getType());
1431 unsigned NumElts = VecTy->getNumElements();
1432
1433 SmallVector<uint32_t, 8> Idxs(NumElts);
1434 for (unsigned i = 0; i != NumElts; ++i)
1435 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
1436
1437 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
1438
1439 if (CI->getNumArgOperands() == 4)
1440 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1441 CI->getArgOperand(2));
1442 } else if (IsX86 && (Name.startswith("avx.vperm2f128.") ||
1443 Name == "avx2.vperm2i128")) {
1444 // The immediate permute control byte looks like this:
1445 // [1:0] - select 128 bits from sources for low half of destination
1446 // [2] - ignore
1447 // [3] - zero low half of destination
1448 // [5:4] - select 128 bits from sources for high half of destination
1449 // [6] - ignore
1450 // [7] - zero high half of destination
1451
1452 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
1453
1454 unsigned NumElts = CI->getType()->getVectorNumElements();
1455 unsigned HalfSize = NumElts / 2;
1456 SmallVector<uint32_t, 8> ShuffleMask(NumElts);
1457
1458 // Determine which operand(s) are actually in use for this instruction.
1459 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
1460 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
1461
1462 // If needed, replace operands based on zero mask.
1463 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
1464 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
1465
1466 // Permute low half of result.
1467 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
1468 for (unsigned i = 0; i < HalfSize; ++i)
1469 ShuffleMask[i] = StartIndex + i;
1470
1471 // Permute high half of result.
1472 StartIndex = (Imm & 0x10) ? HalfSize : 0;
1473 for (unsigned i = 0; i < HalfSize; ++i)
1474 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
1475
1476 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
1477
1478 } else if (IsX86 && (Name.startswith("avx.vpermil.") ||
1479 Name == "sse2.pshuf.d" ||
1480 Name.startswith("avx512.mask.vpermil.p") ||
1481 Name.startswith("avx512.mask.pshuf.d."))) {
1482 Value *Op0 = CI->getArgOperand(0);
1483 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
1484 VectorType *VecTy = cast<VectorType>(CI->getType());
1485 unsigned NumElts = VecTy->getNumElements();
1486 // Calculate the size of each index in the immediate.
1487 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
1488 unsigned IdxMask = ((1 << IdxSize) - 1);
1489
1490 SmallVector<uint32_t, 8> Idxs(NumElts);
1491 // Lookup the bits for this element, wrapping around the immediate every
1492 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
1493 // to offset by the first index of each group.
1494 for (unsigned i = 0; i != NumElts; ++i)
1495 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
1496
1497 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
1498
1499 if (CI->getNumArgOperands() == 4)
1500 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1501 CI->getArgOperand(2));
1502 } else if (IsX86 && (Name == "sse2.pshufl.w" ||
1503 Name.startswith("avx512.mask.pshufl.w."))) {
1504 Value *Op0 = CI->getArgOperand(0);
1505 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
1506 unsigned NumElts = CI->getType()->getVectorNumElements();
1507
1508 SmallVector<uint32_t, 16> Idxs(NumElts);
1509 for (unsigned l = 0; l != NumElts; l += 8) {
1510 for (unsigned i = 0; i != 4; ++i)
1511 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
1512 for (unsigned i = 4; i != 8; ++i)
1513 Idxs[i + l] = i + l;
1514 }
1515
1516 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
1517
1518 if (CI->getNumArgOperands() == 4)
1519 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1520 CI->getArgOperand(2));
1521 } else if (IsX86 && (Name == "sse2.pshufh.w" ||
1522 Name.startswith("avx512.mask.pshufh.w."))) {
1523 Value *Op0 = CI->getArgOperand(0);
1524 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
1525 unsigned NumElts = CI->getType()->getVectorNumElements();
1526
1527 SmallVector<uint32_t, 16> Idxs(NumElts);
1528 for (unsigned l = 0; l != NumElts; l += 8) {
1529 for (unsigned i = 0; i != 4; ++i)
1530 Idxs[i + l] = i + l;
1531 for (unsigned i = 0; i != 4; ++i)
1532 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
1533 }
1534
1535 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
1536
1537 if (CI->getNumArgOperands() == 4)
1538 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1539 CI->getArgOperand(2));
1540 } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {
1541 Value *Op0 = CI->getArgOperand(0);
1542 Value *Op1 = CI->getArgOperand(1);
1543 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
1544 unsigned NumElts = CI->getType()->getVectorNumElements();
1545
1546 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
1547 unsigned HalfLaneElts = NumLaneElts / 2;
1548
1549 SmallVector<uint32_t, 16> Idxs(NumElts);
1550 for (unsigned i = 0; i != NumElts; ++i) {
1551 // Base index is the starting element of the lane.
1552 Idxs[i] = i - (i % NumLaneElts);
1553 // If we are half way through the lane switch to the other source.
1554 if ((i % NumLaneElts) >= HalfLaneElts)
1555 Idxs[i] += NumElts;
1556 // Now select the specific element. By adding HalfLaneElts bits from
1557 // the immediate. Wrapping around the immediate every 8-bits.
1558 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
1559 }
1560
1561 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
1562
1563 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
1564 CI->getArgOperand(3));
1565 } else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||
1566 Name.startswith("avx512.mask.movshdup") ||
1567 Name.startswith("avx512.mask.movsldup"))) {
1568 Value *Op0 = CI->getArgOperand(0);
1569 unsigned NumElts = CI->getType()->getVectorNumElements();
1570 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
1571
1572 unsigned Offset = 0;
1573 if (Name.startswith("avx512.mask.movshdup."))
1574 Offset = 1;
1575
1576 SmallVector<uint32_t, 16> Idxs(NumElts);
1577 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
1578 for (unsigned i = 0; i != NumLaneElts; i += 2) {
1579 Idxs[i + l + 0] = i + l + Offset;
1580 Idxs[i + l + 1] = i + l + Offset;
1581 }
1582
1583 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
1584
1585 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1586 CI->getArgOperand(1));
1587 } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||
1588 Name.startswith("avx512.mask.unpckl."))) {
1589 Value *Op0 = CI->getArgOperand(0);
1590 Value *Op1 = CI->getArgOperand(1);
1591 int NumElts = CI->getType()->getVectorNumElements();
1592 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
1593
1594 SmallVector<uint32_t, 64> Idxs(NumElts);
1595 for (int l = 0; l != NumElts; l += NumLaneElts)
1596 for (int i = 0; i != NumLaneElts; ++i)
1597 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
1598
1599 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
1600
1601 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1602 CI->getArgOperand(2));
1603 } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||
1604 Name.startswith("avx512.mask.unpckh."))) {
1605 Value *Op0 = CI->getArgOperand(0);
1606 Value *Op1 = CI->getArgOperand(1);
1607 int NumElts = CI->getType()->getVectorNumElements();
1608 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
1609
1610 SmallVector<uint32_t, 64> Idxs(NumElts);
1611 for (int l = 0; l != NumElts; l += NumLaneElts)
1612 for (int i = 0; i != NumLaneElts; ++i)
1613 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
1614
1615 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
1616
1617 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1618 CI->getArgOperand(2));
1619 } else if (IsX86 && Name.startswith("avx512.mask.pand.")) {
1620 Rep = Builder.CreateAnd(CI->getArgOperand(0), CI->getArgOperand(1));
1621 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1622 CI->getArgOperand(2));
1623 } else if (IsX86 && Name.startswith("avx512.mask.pandn.")) {
1624 Rep = Builder.CreateAnd(Builder.CreateNot(CI->getArgOperand(0)),
1625 CI->getArgOperand(1));
1626 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1627 CI->getArgOperand(2));
1628 } else if (IsX86 && Name.startswith("avx512.mask.por.")) {
1629 Rep = Builder.CreateOr(CI->getArgOperand(0), CI->getArgOperand(1));
1630 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1631 CI->getArgOperand(2));
1632 } else if (IsX86 && Name.startswith("avx512.mask.pxor.")) {
1633 Rep = Builder.CreateXor(CI->getArgOperand(0), CI->getArgOperand(1));
1634 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1635 CI->getArgOperand(2));
1636 } else if (IsX86 && Name.startswith("avx512.mask.and.")) {
1637 VectorType *FTy = cast<VectorType>(CI->getType());
1638 VectorType *ITy = VectorType::getInteger(FTy);
1639 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
1640 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
1641 Rep = Builder.CreateBitCast(Rep, FTy);
1642 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1643 CI->getArgOperand(2));
1644 } else if (IsX86 && Name.startswith("avx512.mask.andn.")) {
1645 VectorType *FTy = cast<VectorType>(CI->getType());
1646 VectorType *ITy = VectorType::getInteger(FTy);
1647 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
1648 Rep = Builder.CreateAnd(Rep,
1649 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
1650 Rep = Builder.CreateBitCast(Rep, FTy);
1651 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1652 CI->getArgOperand(2));
1653 } else if (IsX86 && Name.startswith("avx512.mask.or.")) {
1654 VectorType *FTy = cast<VectorType>(CI->getType());
1655 VectorType *ITy = VectorType::getInteger(FTy);
1656 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
1657 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
1658 Rep = Builder.CreateBitCast(Rep, FTy);
1659 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1660 CI->getArgOperand(2));
1661 } else if (IsX86 && Name.startswith("avx512.mask.xor.")) {
1662 VectorType *FTy = cast<VectorType>(CI->getType());
1663 VectorType *ITy = VectorType::getInteger(FTy);
1664 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
1665 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
1666 Rep = Builder.CreateBitCast(Rep, FTy);
1667 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1668 CI->getArgOperand(2));
1669 } else if (IsX86 && Name.startswith("avx512.mask.padd.")) {
1670 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
1671 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1672 CI->getArgOperand(2));
1673 } else if (IsX86 && Name.startswith("avx512.mask.psub.")) {
1674 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
1675 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1676 CI->getArgOperand(2));
1677 } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {
1678 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
1679 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1680 CI->getArgOperand(2));
1681 } else if (IsX86 && (Name.startswith("avx512.mask.add.p"))) {
1682 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
1683 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1684 CI->getArgOperand(2));
1685 } else if (IsX86 && Name.startswith("avx512.mask.div.p")) {
1686 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
1687 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1688 CI->getArgOperand(2));
1689 } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {
1690 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
1691 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1692 CI->getArgOperand(2));
1693 } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {
1694 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
1695 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1696 CI->getArgOperand(2));
1697 } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) {
1698 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1699 Intrinsic::ctlz,
1700 CI->getType()),
1701 { CI->getArgOperand(0), Builder.getInt1(false) });
1702 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1703 CI->getArgOperand(1));
1704 } else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
1705 Name.startswith("avx512.mask.min.p"))) {
1706 bool IsMin = Name[13] == 'i';
1707 VectorType *VecTy = cast<VectorType>(CI->getType());
1708 unsigned VecWidth = VecTy->getPrimitiveSizeInBits();
1709 unsigned EltWidth = VecTy->getScalarSizeInBits();
1710 Intrinsic::ID IID;
1711 if (!IsMin && VecWidth == 128 && EltWidth == 32)
1712 IID = Intrinsic::x86_sse_max_ps;
1713 else if (!IsMin && VecWidth == 128 && EltWidth == 64)
1714 IID = Intrinsic::x86_sse2_max_pd;
1715 else if (!IsMin && VecWidth == 256 && EltWidth == 32)
1716 IID = Intrinsic::x86_avx_max_ps_256;
1717 else if (!IsMin && VecWidth == 256 && EltWidth == 64)
1718 IID = Intrinsic::x86_avx_max_pd_256;
1719 else if (IsMin && VecWidth == 128 && EltWidth == 32)
1720 IID = Intrinsic::x86_sse_min_ps;
1721 else if (IsMin && VecWidth == 128 && EltWidth == 64)
1722 IID = Intrinsic::x86_sse2_min_pd;
1723 else if (IsMin && VecWidth == 256 && EltWidth == 32)
1724 IID = Intrinsic::x86_avx_min_ps_256;
1725 else if (IsMin && VecWidth == 256 && EltWidth == 64)
1726 IID = Intrinsic::x86_avx_min_pd_256;
1727 else
1728 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1728);
1729
1730 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
1731 { CI->getArgOperand(0), CI->getArgOperand(1) });
1732 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1733 CI->getArgOperand(2));
1734 } else if (IsX86 && Name.startswith("avx512.mask.pshuf.b.")) {
1735 VectorType *VecTy = cast<VectorType>(CI->getType());
1736 Intrinsic::ID IID;
1737 if (VecTy->getPrimitiveSizeInBits() == 128)
1738 IID = Intrinsic::x86_ssse3_pshuf_b_128;
1739 else if (VecTy->getPrimitiveSizeInBits() == 256)
1740 IID = Intrinsic::x86_avx2_pshuf_b;
1741 else if (VecTy->getPrimitiveSizeInBits() == 512)
1742 IID = Intrinsic::x86_avx512_pshuf_b_512;
1743 else
1744 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1744);
1745
1746 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
1747 { CI->getArgOperand(0), CI->getArgOperand(1) });
1748 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1749 CI->getArgOperand(2));
1750 } else if (IsX86 && (Name.startswith("avx512.mask.pmul.dq.") ||
1751 Name.startswith("avx512.mask.pmulu.dq."))) {
1752 bool IsUnsigned = Name[16] == 'u';
1753 VectorType *VecTy = cast<VectorType>(CI->getType());
1754 Intrinsic::ID IID;
1755 if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 128)
1756 IID = Intrinsic::x86_sse41_pmuldq;
1757 else if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 256)
1758 IID = Intrinsic::x86_avx2_pmul_dq;
1759 else if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 512)
1760 IID = Intrinsic::x86_avx512_pmul_dq_512;
1761 else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 128)
1762 IID = Intrinsic::x86_sse2_pmulu_dq;
1763 else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 256)
1764 IID = Intrinsic::x86_avx2_pmulu_dq;
1765 else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 512)
1766 IID = Intrinsic::x86_avx512_pmulu_dq_512;
1767 else
1768 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1768);
1769
1770 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
1771 { CI->getArgOperand(0), CI->getArgOperand(1) });
1772 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1773 CI->getArgOperand(2));
1774 } else if (IsX86 && Name.startswith("avx512.mask.pack")) {
1775 bool IsUnsigned = Name[16] == 'u';
1776 bool IsDW = Name[18] == 'd';
1777 VectorType *VecTy = cast<VectorType>(CI->getType());
1778 Intrinsic::ID IID;
1779 if (!IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 128)
1780 IID = Intrinsic::x86_sse2_packsswb_128;
1781 else if (!IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 256)
1782 IID = Intrinsic::x86_avx2_packsswb;
1783 else if (!IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 512)
1784 IID = Intrinsic::x86_avx512_packsswb_512;
1785 else if (!IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 128)
1786 IID = Intrinsic::x86_sse2_packssdw_128;
1787 else if (!IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 256)
1788 IID = Intrinsic::x86_avx2_packssdw;
1789 else if (!IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 512)
1790 IID = Intrinsic::x86_avx512_packssdw_512;
1791 else if (IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 128)
1792 IID = Intrinsic::x86_sse2_packuswb_128;
1793 else if (IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 256)
1794 IID = Intrinsic::x86_avx2_packuswb;
1795 else if (IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 512)
1796 IID = Intrinsic::x86_avx512_packuswb_512;
1797 else if (IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 128)
1798 IID = Intrinsic::x86_sse41_packusdw;
1799 else if (IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 256)
1800 IID = Intrinsic::x86_avx2_packusdw;
1801 else if (IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 512)
1802 IID = Intrinsic::x86_avx512_packusdw_512;
1803 else
1804 llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1804);
1805
1806 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
1807 { CI->getArgOperand(0), CI->getArgOperand(1) });
1808 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
1809 CI->getArgOperand(2));
1810 } else if (IsX86 && Name.startswith("avx512.mask.psll")) {
1811 bool IsImmediate = Name[16] == 'i' ||
1812 (Name.size() > 18 && Name[18] == 'i');
1813 bool IsVariable = Name[16] == 'v';
1814 char Size = Name[16] == '.' ? Name[17] :
1815 Name[17] == '.' ? Name[18] :
1816 Name[18] == '.' ? Name[19] :
1817 Name[20];
1818
1819 Intrinsic::ID IID;
1820 if (IsVariable && Name[17] != '.') {
1821 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
1822 IID = Intrinsic::x86_avx2_psllv_q;
1823 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
1824 IID = Intrinsic::x86_avx2_psllv_q_256;
1825 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
1826 IID = Intrinsic::x86_avx2_psllv_d;
1827 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
1828 IID = Intrinsic::x86_avx2_psllv_d_256;
1829 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
1830 IID = Intrinsic::x86_avx512_psllv_w_128;
1831 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
1832 IID = Intrinsic::x86_avx512_psllv_w_256;
1833 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
1834 IID = Intrinsic::x86_avx512_psllv_w_512;
1835 else
1836 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1836);
1837 } else if (Name.endswith(".128")) {
1838 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
1839 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
1840 : Intrinsic::x86_sse2_psll_d;
1841 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
1842 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
1843 : Intrinsic::x86_sse2_psll_q;
1844 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
1845 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
1846 : Intrinsic::x86_sse2_psll_w;
1847 else
1848 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1848);
1849 } else if (Name.endswith(".256")) {
1850 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
1851 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
1852 : Intrinsic::x86_avx2_psll_d;
1853 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
1854 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
1855 : Intrinsic::x86_avx2_psll_q;
1856 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
1857 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
1858 : Intrinsic::x86_avx2_psll_w;
1859 else
1860 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1860);
1861 } else {
1862 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
1863 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
1864 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
1865 Intrinsic::x86_avx512_psll_d_512;
1866 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
1867 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
1868 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
1869 Intrinsic::x86_avx512_psll_q_512;
1870 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
1871 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
1872 : Intrinsic::x86_avx512_psll_w_512;
1873 else
1874 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1874);
1875 }
1876
1877 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
1878 } else if (IsX86 && Name.startswith("avx512.mask.psrl")) {
1879 bool IsImmediate = Name[16] == 'i' ||
1880 (Name.size() > 18 && Name[18] == 'i');
1881 bool IsVariable = Name[16] == 'v';
1882 char Size = Name[16] == '.' ? Name[17] :
1883 Name[17] == '.' ? Name[18] :
1884 Name[18] == '.' ? Name[19] :
1885 Name[20];
1886
1887 Intrinsic::ID IID;
1888 if (IsVariable && Name[17] != '.') {
1889 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
1890 IID = Intrinsic::x86_avx2_psrlv_q;
1891 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
1892 IID = Intrinsic::x86_avx2_psrlv_q_256;
1893 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
1894 IID = Intrinsic::x86_avx2_psrlv_d;
1895 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
1896 IID = Intrinsic::x86_avx2_psrlv_d_256;
1897 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
1898 IID = Intrinsic::x86_avx512_psrlv_w_128;
1899 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
1900 IID = Intrinsic::x86_avx512_psrlv_w_256;
1901 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
1902 IID = Intrinsic::x86_avx512_psrlv_w_512;
1903 else
1904 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1904);
1905 } else if (Name.endswith(".128")) {
1906 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
1907 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
1908 : Intrinsic::x86_sse2_psrl_d;
1909 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
1910 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
1911 : Intrinsic::x86_sse2_psrl_q;
1912 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
1913 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
1914 : Intrinsic::x86_sse2_psrl_w;
1915 else
1916 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1916);
1917 } else if (Name.endswith(".256")) {
1918 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
1919 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
1920 : Intrinsic::x86_avx2_psrl_d;
1921 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
1922 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
1923 : Intrinsic::x86_avx2_psrl_q;
1924 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
1925 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
1926 : Intrinsic::x86_avx2_psrl_w;
1927 else
1928 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1928);
1929 } else {
1930 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
1931 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
1932 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
1933 Intrinsic::x86_avx512_psrl_d_512;
1934 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
1935 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
1936 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
1937 Intrinsic::x86_avx512_psrl_q_512;
1938 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
1939 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
1940 : Intrinsic::x86_avx512_psrl_w_512;
1941 else
1942 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1942);
1943 }
1944
1945 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
1946 } else if (IsX86 && Name.startswith("avx512.mask.psra")) {
1947 bool IsImmediate = Name[16] == 'i' ||
1948 (Name.size() > 18 && Name[18] == 'i');
1949 bool IsVariable = Name[16] == 'v';
1950 char Size = Name[16] == '.' ? Name[17] :
1951 Name[17] == '.' ? Name[18] :
1952 Name[18] == '.' ? Name[19] :
1953 Name[20];
1954
1955 Intrinsic::ID IID;
1956 if (IsVariable && Name[17] != '.') {
1957 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
1958 IID = Intrinsic::x86_avx2_psrav_d;
1959 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
1960 IID = Intrinsic::x86_avx2_psrav_d_256;
1961 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
1962 IID = Intrinsic::x86_avx512_psrav_w_128;
1963 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
1964 IID = Intrinsic::x86_avx512_psrav_w_256;
1965 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
1966 IID = Intrinsic::x86_avx512_psrav_w_512;
1967 else
1968 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1968);
1969 } else if (Name.endswith(".128")) {
1970 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
1971 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
1972 : Intrinsic::x86_sse2_psra_d;
1973 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
1974 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
1975 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
1976 Intrinsic::x86_avx512_psra_q_128;
1977 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
1978 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
1979 : Intrinsic::x86_sse2_psra_w;
1980 else
1981 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1981);
1982 } else if (Name.endswith(".256")) {
1983 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
1984 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
1985 : Intrinsic::x86_avx2_psra_d;
1986 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
1987 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
1988 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
1989 Intrinsic::x86_avx512_psra_q_256;
1990 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
1991 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
1992 : Intrinsic::x86_avx2_psra_w;
1993 else
1994 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 1994);
1995 } else {
1996 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
1997 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
1998 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
1999 Intrinsic::x86_avx512_psra_d_512;
2000 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
2001 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
2002 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
2003 Intrinsic::x86_avx512_psra_q_512;
2004 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
2005 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
2006 : Intrinsic::x86_avx512_psra_w_512;
2007 else
2008 llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2008);
2009 }
2010
2011 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
2012 } else if (IsX86 && Name.startswith("avx512.mask.move.s")) {
2013 Rep = upgradeMaskedMove(Builder, *CI);
2014 } else if (IsX86 && Name.startswith("avx512.cvtmask2")) {
2015 Rep = UpgradeMaskToInt(Builder, *CI);
2016 } else if (IsX86 && Name.startswith("avx512.mask.vpermilvar.")) {
2017 Intrinsic::ID IID;
2018 if (Name.endswith("ps.128"))
2019 IID = Intrinsic::x86_avx_vpermilvar_ps;
2020 else if (Name.endswith("pd.128"))
2021 IID = Intrinsic::x86_avx_vpermilvar_pd;
2022 else if (Name.endswith("ps.256"))
2023 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2024 else if (Name.endswith("pd.256"))
2025 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2026 else if (Name.endswith("ps.512"))
2027 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2028 else if (Name.endswith("pd.512"))
2029 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2030 else
2031 llvm_unreachable("Unexpected vpermilvar intrinsic")::llvm::llvm_unreachable_internal("Unexpected vpermilvar intrinsic"
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2031);
2032
2033 Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID);
2034 Rep = Builder.CreateCall(Intrin,
2035 { CI->getArgOperand(0), CI->getArgOperand(1) });
2036 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2037 CI->getArgOperand(2));
2038 } else if (IsX86 && Name.endswith(".movntdqa")) {
2039 Module *M = F->getParent();
2040 MDNode *Node = MDNode::get(
2041 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
2042
2043 Value *Ptr = CI->getArgOperand(0);
2044 VectorType *VTy = cast<VectorType>(CI->getType());
2045
2046 // Convert the type of the pointer to a pointer to the stored type.
2047 Value *BC =
2048 Builder.CreateBitCast(Ptr, PointerType::getUnqual(VTy), "cast");
2049 LoadInst *LI = Builder.CreateAlignedLoad(BC, VTy->getBitWidth() / 8);
2050 LI->setMetadata(M->getMDKindID("nontemporal"), Node);
2051 Rep = LI;
2052 } else if (IsX86 &&
2053 (Name.startswith("sse2.pavg") || Name.startswith("avx2.pavg") ||
2054 Name.startswith("avx512.mask.pavg"))) {
2055 // llvm.x86.sse2.pavg.b/w, llvm.x86.avx2.pavg.b/w,
2056 // llvm.x86.avx512.mask.pavg.b/w
2057 Value *A = CI->getArgOperand(0);
2058 Value *B = CI->getArgOperand(1);
2059 VectorType *ZextType = VectorType::getExtendedElementVectorType(
2060 cast<VectorType>(A->getType()));
2061 Value *ExtendedA = Builder.CreateZExt(A, ZextType);
2062 Value *ExtendedB = Builder.CreateZExt(B, ZextType);
2063 Value *Sum = Builder.CreateAdd(ExtendedA, ExtendedB);
2064 Value *AddOne = Builder.CreateAdd(Sum, ConstantInt::get(ZextType, 1));
2065 Value *ShiftR = Builder.CreateLShr(AddOne, ConstantInt::get(ZextType, 1));
2066 Rep = Builder.CreateTrunc(ShiftR, A->getType());
2067 if (CI->getNumArgOperands() > 2) {
2068 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2069 CI->getArgOperand(2));
2070 }
2071 } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {
2072 Value *Arg = CI->getArgOperand(0);
2073 Value *Neg = Builder.CreateNeg(Arg, "neg");
2074 Value *Cmp = Builder.CreateICmpSGE(
2075 Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
2076 Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
2077 } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
2078 Name == "max.ui" || Name == "max.ull")) {
2079 Value *Arg0 = CI->getArgOperand(0);
2080 Value *Arg1 = CI->getArgOperand(1);
2081 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
2082 ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
2083 : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
2084 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
2085 } else if (IsNVVM && (Name == "min.i" || Name == "min.ll" ||
2086 Name == "min.ui" || Name == "min.ull")) {
2087 Value *Arg0 = CI->getArgOperand(0);
2088 Value *Arg1 = CI->getArgOperand(1);
2089 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
2090 ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
2091 : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
2092 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
2093 } else if (IsNVVM && Name == "clz.ll") {
2094 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64.
2095 Value *Arg = CI->getArgOperand(0);
2096 Value *Ctlz = Builder.CreateCall(
2097 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
2098 {Arg->getType()}),
2099 {Arg, Builder.getFalse()}, "ctlz");
2100 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
2101 } else if (IsNVVM && Name == "popc.ll") {
2102 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an
2103 // i64.
2104 Value *Arg = CI->getArgOperand(0);
2105 Value *Popc = Builder.CreateCall(
2106 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
2107 {Arg->getType()}),
2108 Arg, "ctpop");
2109 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
2110 } else if (IsNVVM && Name == "h2f") {
2111 Rep = Builder.CreateCall(Intrinsic::getDeclaration(
2112 F->getParent(), Intrinsic::convert_from_fp16,
2113 {Builder.getFloatTy()}),
2114 CI->getArgOperand(0), "h2f");
2115 } else {
2116 llvm_unreachable("Unknown function for CallInst upgrade.")::llvm::llvm_unreachable_internal("Unknown function for CallInst upgrade."
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2116);
2117 }
2118
2119 if (Rep)
2120 CI->replaceAllUsesWith(Rep);
2121 CI->eraseFromParent();
2122 return;
2123 }
2124
2125 CallInst *NewCall = nullptr;
2126 switch (NewFn->getIntrinsicID()) {
2127 default: {
2128 // Handle generic mangling change, but nothing else
2129 assert((((CI->getCalledFunction()->getName() != NewFn->getName
()) && "Unknown function for CallInst upgrade and isn't just a name change"
) ? static_cast<void> (0) : __assert_fail ("(CI->getCalledFunction()->getName() != NewFn->getName()) && \"Unknown function for CallInst upgrade and isn't just a name change\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2131, __PRETTY_FUNCTION__))
2130 (CI->getCalledFunction()->getName() != NewFn->getName()) &&(((CI->getCalledFunction()->getName() != NewFn->getName
()) && "Unknown function for CallInst upgrade and isn't just a name change"
) ? static_cast<void> (0) : __assert_fail ("(CI->getCalledFunction()->getName() != NewFn->getName()) && \"Unknown function for CallInst upgrade and isn't just a name change\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2131, __PRETTY_FUNCTION__))
2131 "Unknown function for CallInst upgrade and isn't just a name change")(((CI->getCalledFunction()->getName() != NewFn->getName
()) && "Unknown function for CallInst upgrade and isn't just a name change"
) ? static_cast<void> (0) : __assert_fail ("(CI->getCalledFunction()->getName() != NewFn->getName()) && \"Unknown function for CallInst upgrade and isn't just a name change\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2131, __PRETTY_FUNCTION__));
2132 CI->setCalledFunction(NewFn);
2133 return;
2134 }
2135
2136 case Intrinsic::arm_neon_vld1:
2137 case Intrinsic::arm_neon_vld2:
2138 case Intrinsic::arm_neon_vld3:
2139 case Intrinsic::arm_neon_vld4:
2140 case Intrinsic::arm_neon_vld2lane:
2141 case Intrinsic::arm_neon_vld3lane:
2142 case Intrinsic::arm_neon_vld4lane:
2143 case Intrinsic::arm_neon_vst1:
2144 case Intrinsic::arm_neon_vst2:
2145 case Intrinsic::arm_neon_vst3:
2146 case Intrinsic::arm_neon_vst4:
2147 case Intrinsic::arm_neon_vst2lane:
2148 case Intrinsic::arm_neon_vst3lane:
2149 case Intrinsic::arm_neon_vst4lane: {
2150 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
2151 CI->arg_operands().end());
2152 NewCall = Builder.CreateCall(NewFn, Args);
2153 break;
2154 }
2155
2156 case Intrinsic::bitreverse:
2157 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
2158 break;
2159
2160 case Intrinsic::ctlz:
2161 case Intrinsic::cttz:
2162 assert(CI->getNumArgOperands() == 1 &&((CI->getNumArgOperands() == 1 && "Mismatch between function args and call args"
) ? static_cast<void> (0) : __assert_fail ("CI->getNumArgOperands() == 1 && \"Mismatch between function args and call args\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2163, __PRETTY_FUNCTION__))
2163 "Mismatch between function args and call args")((CI->getNumArgOperands() == 1 && "Mismatch between function args and call args"
) ? static_cast<void> (0) : __assert_fail ("CI->getNumArgOperands() == 1 && \"Mismatch between function args and call args\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2163, __PRETTY_FUNCTION__));
2164 NewCall =
2165 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
2166 break;
2167
2168 case Intrinsic::objectsize: {
2169 Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
2170 ? Builder.getFalse()
2171 : CI->getArgOperand(2);
2172 NewCall = Builder.CreateCall(
2173 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize});
2174 break;
2175 }
2176
2177 case Intrinsic::ctpop:
2178 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
2179 break;
2180
2181 case Intrinsic::convert_from_fp16:
2182 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
2183 break;
2184
2185 case Intrinsic::dbg_value:
2186 // Upgrade from the old version that had an extra offset argument.
2187 assert(CI->getNumArgOperands() == 4)((CI->getNumArgOperands() == 4) ? static_cast<void> (
0) : __assert_fail ("CI->getNumArgOperands() == 4", "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2187, __PRETTY_FUNCTION__));
2188 // Drop nonzero offsets instead of attempting to upgrade them.
2189 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
2190 if (Offset->isZeroValue()) {
2191 NewCall = Builder.CreateCall(
2192 NewFn,
2193 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
2194 break;
2195 }
2196 CI->eraseFromParent();
2197 return;
2198
2199 case Intrinsic::x86_xop_vfrcz_ss:
2200 case Intrinsic::x86_xop_vfrcz_sd:
2201 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
2202 break;
2203
2204 case Intrinsic::x86_xop_vpermil2pd:
2205 case Intrinsic::x86_xop_vpermil2ps:
2206 case Intrinsic::x86_xop_vpermil2pd_256:
2207 case Intrinsic::x86_xop_vpermil2ps_256: {
2208 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
2209 CI->arg_operands().end());
2210 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
2211 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
2212 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
2213 NewCall = Builder.CreateCall(NewFn, Args);
2214 break;
2215 }
2216
2217 case Intrinsic::x86_sse41_ptestc:
2218 case Intrinsic::x86_sse41_ptestz:
2219 case Intrinsic::x86_sse41_ptestnzc: {
2220 // The arguments for these intrinsics used to be v4f32, and changed
2221 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
2222 // So, the only thing required is a bitcast for both arguments.
2223 // First, check the arguments have the old type.
2224 Value *Arg0 = CI->getArgOperand(0);
2225 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
2226 return;
2227
2228 // Old intrinsic, add bitcasts
2229 Value *Arg1 = CI->getArgOperand(1);
2230
2231 Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2);
2232
2233 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
2234 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
2235
2236 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
2237 break;
2238 }
2239
2240 case Intrinsic::x86_sse41_insertps:
2241 case Intrinsic::x86_sse41_dppd:
2242 case Intrinsic::x86_sse41_dpps:
2243 case Intrinsic::x86_sse41_mpsadbw:
2244 case Intrinsic::x86_avx_dp_ps_256:
2245 case Intrinsic::x86_avx2_mpsadbw: {
2246 // Need to truncate the last argument from i32 to i8 -- this argument models
2247 // an inherently 8-bit immediate operand to these x86 instructions.
2248 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
2249 CI->arg_operands().end());
2250
2251 // Replace the last argument with a trunc.
2252 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
2253 NewCall = Builder.CreateCall(NewFn, Args);
2254 break;
2255 }
2256
2257 case Intrinsic::thread_pointer: {
2258 NewCall = Builder.CreateCall(NewFn, {});
2259 break;
2260 }
2261
2262 case Intrinsic::invariant_start:
2263 case Intrinsic::invariant_end:
2264 case Intrinsic::masked_load:
2265 case Intrinsic::masked_store:
2266 case Intrinsic::masked_gather:
2267 case Intrinsic::masked_scatter: {
2268 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
2269 CI->arg_operands().end());
2270 NewCall = Builder.CreateCall(NewFn, Args);
2271 break;
2272 }
2273 }
2274 assert(NewCall && "Should have either set this variable or returned through "((NewCall && "Should have either set this variable or returned through "
"the default case") ? static_cast<void> (0) : __assert_fail
("NewCall && \"Should have either set this variable or returned through \" \"the default case\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2275, __PRETTY_FUNCTION__))
2275 "the default case")((NewCall && "Should have either set this variable or returned through "
"the default case") ? static_cast<void> (0) : __assert_fail
("NewCall && \"Should have either set this variable or returned through \" \"the default case\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2275, __PRETTY_FUNCTION__));
2276 std::string Name = CI->getName();
2277 if (!Name.empty()) {
2278 CI->setName(Name + ".old");
2279 NewCall->setName(Name);
2280 }
2281 CI->replaceAllUsesWith(NewCall);
2282 CI->eraseFromParent();
2283}
2284
2285void llvm::UpgradeCallsToIntrinsic(Function *F) {
2286 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.")((F && "Illegal attempt to upgrade a non-existent intrinsic."
) ? static_cast<void> (0) : __assert_fail ("F && \"Illegal attempt to upgrade a non-existent intrinsic.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2286, __PRETTY_FUNCTION__));
2287
2288 // Check if this function should be upgraded and get the replacement function
2289 // if there is one.
2290 Function *NewFn;
2291 if (UpgradeIntrinsicFunction(F, NewFn)) {
2292 // Replace all users of the old function with the new function or new
2293 // instructions. This is not a range loop because the call is deleted.
2294 for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE; )
2295 if (CallInst *CI = dyn_cast<CallInst>(*UI++))
2296 UpgradeIntrinsicCall(CI, NewFn);
2297
2298 // Remove old function, no longer used, from the module.
2299 F->eraseFromParent();
2300 }
2301}
2302
2303MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
2304 // Check if the tag uses struct-path aware TBAA format.
2305 if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3)
2306 return &MD;
2307
2308 auto &Context = MD.getContext();
2309 if (MD.getNumOperands() == 3) {
2310 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
2311 MDNode *ScalarType = MDNode::get(Context, Elts);
2312 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
2313 Metadata *Elts2[] = {ScalarType, ScalarType,
2314 ConstantAsMetadata::get(
2315 Constant::getNullValue(Type::getInt64Ty(Context))),
2316 MD.getOperand(2)};
2317 return MDNode::get(Context, Elts2);
2318 }
2319 // Create a MDNode <MD, MD, offset 0>
2320 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
2321 Type::getInt64Ty(Context)))};
2322 return MDNode::get(Context, Elts);
2323}
2324
2325Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
2326 Instruction *&Temp) {
2327 if (Opc != Instruction::BitCast)
2328 return nullptr;
2329
2330 Temp = nullptr;
2331 Type *SrcTy = V->getType();
2332 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
2333 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
2334 LLVMContext &Context = V->getContext();
2335
2336 // We have no information about target data layout, so we assume that
2337 // the maximum pointer size is 64bit.
2338 Type *MidTy = Type::getInt64Ty(Context);
2339 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
2340
2341 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
2342 }
2343
2344 return nullptr;
2345}
2346
2347Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
2348 if (Opc != Instruction::BitCast)
2349 return nullptr;
2350
2351 Type *SrcTy = C->getType();
2352 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
2353 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
2354 LLVMContext &Context = C->getContext();
2355
2356 // We have no information about target data layout, so we assume that
2357 // the maximum pointer size is 64bit.
2358 Type *MidTy = Type::getInt64Ty(Context);
2359
2360 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
2361 DestTy);
2362 }
2363
2364 return nullptr;
2365}
2366
2367/// Check the debug info version number, if it is out-dated, drop the debug
2368/// info. Return true if module is modified.
2369bool llvm::UpgradeDebugInfo(Module &M) {
2370 unsigned Version = getDebugMetadataVersionFromModule(M);
2371 if (Version == DEBUG_METADATA_VERSION) {
2372 bool BrokenDebugInfo = false;
2373 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
2374 report_fatal_error("Broken module found, compilation aborted!");
2375 if (!BrokenDebugInfo)
2376 // Everything is ok.
2377 return false;
2378 else {
2379 // Diagnose malformed debug info.
2380 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
2381 M.getContext().diagnose(Diag);
2382 }
2383 }
2384 bool Modified = StripDebugInfo(M);
2385 if (Modified && Version != DEBUG_METADATA_VERSION) {
2386 // Diagnose a version mismatch.
2387 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
2388 M.getContext().diagnose(DiagVersion);
2389 }
2390 return Modified;
2391}
2392
2393bool llvm::UpgradeModuleFlags(Module &M) {
2394 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
2395 if (!ModFlags)
2396 return false;
2397
2398 bool HasObjCFlag = false, HasClassProperties = false, Changed = false;
2399 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
2400 MDNode *Op = ModFlags->getOperand(I);
2401 if (Op->getNumOperands() != 3)
2402 continue;
2403 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
2404 if (!ID)
2405 continue;
2406 if (ID->getString() == "Objective-C Image Info Version")
2407 HasObjCFlag = true;
2408 if (ID->getString() == "Objective-C Class Properties")
2409 HasClassProperties = true;
2410 // Upgrade PIC/PIE Module Flags. The module flag behavior for these two
2411 // field was Error and now they are Max.
2412 if (ID->getString() == "PIC Level" || ID->getString() == "PIE Level") {
2413 if (auto *Behavior =
2414 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
2415 if (Behavior->getLimitedValue() == Module::Error) {
2416 Type *Int32Ty = Type::getInt32Ty(M.getContext());
2417 Metadata *Ops[3] = {
2418 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Max)),
2419 MDString::get(M.getContext(), ID->getString()),
2420 Op->getOperand(2)};
2421 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
2422 Changed = true;
2423 }
2424 }
2425 }
2426 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
2427 // section name so that llvm-lto will not complain about mismatching
2428 // module flags that is functionally the same.
2429 if (ID->getString() == "Objective-C Image Info Section") {
2430 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
2431 SmallVector<StringRef, 4> ValueComp;
2432 Value->getString().split(ValueComp, " ");
2433 if (ValueComp.size() != 1) {
2434 std::string NewValue;
2435 for (auto &S : ValueComp)
2436 NewValue += S.str();
2437 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
2438 MDString::get(M.getContext(), NewValue)};
2439 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
2440 Changed = true;
2441 }
2442 }
2443 }
2444 }
2445
2446 // "Objective-C Class Properties" is recently added for Objective-C. We
2447 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
2448 // flag of value 0, so we can correclty downgrade this flag when trying to
2449 // link an ObjC bitcode without this module flag with an ObjC bitcode with
2450 // this module flag.
2451 if (HasObjCFlag && !HasClassProperties) {
2452 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
2453 (uint32_t)0);
2454 Changed = true;
2455 }
2456
2457 return Changed;
2458}
2459
2460void llvm::UpgradeSectionAttributes(Module &M) {
2461 auto TrimSpaces = [](StringRef Section) -> std::string {
2462 SmallVector<StringRef, 5> Components;
2463 Section.split(Components, ',');
2464
2465 SmallString<32> Buffer;
2466 raw_svector_ostream OS(Buffer);
2467
2468 for (auto Component : Components)
2469 OS << ',' << Component.trim();
2470
2471 return OS.str().substr(1);
2472 };
2473
2474 for (auto &GV : M.globals()) {
2475 if (!GV.hasSection())
2476 continue;
2477
2478 StringRef Section = GV.getSection();
2479
2480 if (!Section.startswith("__DATA, __objc_catlist"))
2481 continue;
2482
2483 // __DATA, __objc_catlist, regular, no_dead_strip
2484 // __DATA,__objc_catlist,regular,no_dead_strip
2485 GV.setSection(TrimSpaces(Section));
2486 }
2487}
2488
2489static bool isOldLoopArgument(Metadata *MD) {
2490 auto *T = dyn_cast_or_null<MDTuple>(MD);
2491 if (!T)
2492 return false;
2493 if (T->getNumOperands() < 1)
2494 return false;
2495 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
2496 if (!S)
2497 return false;
2498 return S->getString().startswith("llvm.vectorizer.");
2499}
2500
2501static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
2502 StringRef OldPrefix = "llvm.vectorizer.";
2503 assert(OldTag.startswith(OldPrefix) && "Expected old prefix")((OldTag.startswith(OldPrefix) && "Expected old prefix"
) ? static_cast<void> (0) : __assert_fail ("OldTag.startswith(OldPrefix) && \"Expected old prefix\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/lib/IR/AutoUpgrade.cpp"
, 2503, __PRETTY_FUNCTION__));
2504
2505 if (OldTag == "llvm.vectorizer.unroll")
2506 return MDString::get(C, "llvm.loop.interleave.count");
2507
2508 return MDString::get(
2509 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
2510 .str());
2511}
2512
2513static Metadata *upgradeLoopArgument(Metadata *MD) {
2514 auto *T = dyn_cast_or_null<MDTuple>(MD);
2515 if (!T)
2516 return MD;
2517 if (T->getNumOperands() < 1)
2518 return MD;
2519 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
2520 if (!OldTag)
2521 return MD;
2522 if (!OldTag->getString().startswith("llvm.vectorizer."))
2523 return MD;
2524
2525 // This has an old tag. Upgrade it.
2526 SmallVector<Metadata *, 8> Ops;
2527 Ops.reserve(T->getNumOperands());
2528 Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString()));
2529 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
2530 Ops.push_back(T->getOperand(I));
2531
2532 return MDTuple::get(T->getContext(), Ops);
2533}
2534
2535MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
2536 auto *T = dyn_cast<MDTuple>(&N);
2537 if (!T)
2538 return &N;
2539
2540 if (none_of(T->operands(), isOldLoopArgument))
2541 return &N;
2542
2543 SmallVector<Metadata *, 8> Ops;
2544 Ops.reserve(T->getNumOperands());
2545 for (Metadata *MD : T->operands())
2546 Ops.push_back(upgradeLoopArgument(MD));
2547
2548 return MDTuple::get(T->getContext(), Ops);
2549}
←

/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h

→
1//===- llvm/DerivedTypes.h - Classes for handling data types ----*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file contains the declarations of classes that represent "derived
11// types". These are things like "arrays of x" or "structure of x, y, z" or
12// "function returning x taking (y,z) as parameters", etc...
13//
14// The implementations of these classes live in the Type.cpp file.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_IR_DERIVEDTYPES_H
19#define LLVM_IR_DERIVEDTYPES_H
20
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/IR/Type.h"
25#include "llvm/Support/Casting.h"
26#include "llvm/Support/Compiler.h"
27#include <cassert>
28#include <cstdint>
29
30namespace llvm {
31
32class Value;
33class APInt;
34class LLVMContext;
35
36/// Class to represent integer types. Note that this class is also used to
37/// represent the built-in integer types: Int1Ty, Int8Ty, Int16Ty, Int32Ty and
38/// Int64Ty.
39/// @brief Integer representation type
40class IntegerType : public Type {
41 friend class LLVMContextImpl;
42
43protected:
44 explicit IntegerType(LLVMContext &C, unsigned NumBits) : Type(C, IntegerTyID){
45 setSubclassData(NumBits);
46 }
47
48public:
49 /// This enum is just used to hold constants we need for IntegerType.
50 enum {
51 MIN_INT_BITS = 1, ///< Minimum number of bits that can be specified
52 MAX_INT_BITS = (1<<24)-1 ///< Maximum number of bits that can be specified
53 ///< Note that bit width is stored in the Type classes SubclassData field
54 ///< which has 24 bits. This yields a maximum bit width of 16,777,215
55 ///< bits.
56 };
57
58 /// This static method is the primary way of constructing an IntegerType.
59 /// If an IntegerType with the same NumBits value was previously instantiated,
60 /// that instance will be returned. Otherwise a new one will be created. Only
61 /// one instance with a given NumBits value is ever created.
62 /// @brief Get or create an IntegerType instance.
63 static IntegerType *get(LLVMContext &C, unsigned NumBits);
64
65 /// @brief Get the number of bits in this IntegerType
66 unsigned getBitWidth() const { return getSubclassData(); }
67
68 /// Return a bitmask with ones set for all of the bits that can be set by an
69 /// unsigned version of this type. This is 0xFF for i8, 0xFFFF for i16, etc.
70 uint64_t getBitMask() const {
71 return ~uint64_t(0UL) >> (64-getBitWidth());
72 }
73
74 /// Return a uint64_t with just the most significant bit set (the sign bit, if
75 /// the value is treated as a signed number).
76 uint64_t getSignBit() const {
77 return 1ULL << (getBitWidth()-1);
78 }
79
80 /// For example, this is 0xFF for an 8 bit integer, 0xFFFF for i16, etc.
81 /// @returns a bit mask with ones set for all the bits of this type.
82 /// @brief Get a bit mask for this type.
83 APInt getMask() const;
84
85 /// This method determines if the width of this IntegerType is a power-of-2
86 /// in terms of 8 bit bytes.
87 /// @returns true if this is a power-of-2 byte width.
88 /// @brief Is this a power-of-2 byte-width IntegerType ?
89 bool isPowerOf2ByteWidth() const;
90
91 /// Methods for support type inquiry through isa, cast, and dyn_cast.
92 static bool classof(const Type *T) {
93 return T->getTypeID() == IntegerTyID;
94 }
95};
96
97unsigned Type::getIntegerBitWidth() const {
98 return cast<IntegerType>(this)->getBitWidth();
99}
100
101/// Class to represent function types
102///
103class FunctionType : public Type {
104 FunctionType(Type *Result, ArrayRef<Type*> Params, bool IsVarArgs);
105
106public:
107 FunctionType(const FunctionType &) = delete;
108 FunctionType &operator=(const FunctionType &) = delete;
109
110 /// This static method is the primary way of constructing a FunctionType.
111 static FunctionType *get(Type *Result,
112 ArrayRef<Type*> Params, bool isVarArg);
113
114 /// Create a FunctionType taking no parameters.
115 static FunctionType *get(Type *Result, bool isVarArg);
116
117 /// Return true if the specified type is valid as a return type.
118 static bool isValidReturnType(Type *RetTy);
119
120 /// Return true if the specified type is valid as an argument type.
121 static bool isValidArgumentType(Type *ArgTy);
122
123 bool isVarArg() const { return getSubclassData()!=0; }
124 Type *getReturnType() const { return ContainedTys[0]; }
125
126 using param_iterator = Type::subtype_iterator;
127
128 param_iterator param_begin() const { return ContainedTys + 1; }
129 param_iterator param_end() const { return &ContainedTys[NumContainedTys]; }
130 ArrayRef<Type *> params() const {
131 return makeArrayRef(param_begin(), param_end());
132 }
133
134 /// Parameter type accessors.
135 Type *getParamType(unsigned i) const { return ContainedTys[i+1]; }
136
137 /// Return the number of fixed parameters this function type requires.
138 /// This does not consider varargs.
139 unsigned getNumParams() const { return NumContainedTys - 1; }
140
141 /// Methods for support type inquiry through isa, cast, and dyn_cast.
142 static bool classof(const Type *T) {
143 return T->getTypeID() == FunctionTyID;
144 }
145};
146static_assert(alignof(FunctionType) >= alignof(Type *),
147 "Alignment sufficient for objects appended to FunctionType");
148
149bool Type::isFunctionVarArg() const {
150 return cast<FunctionType>(this)->isVarArg();
151}
152
153Type *Type::getFunctionParamType(unsigned i) const {
154 return cast<FunctionType>(this)->getParamType(i);
155}
156
157unsigned Type::getFunctionNumParams() const {
158 return cast<FunctionType>(this)->getNumParams();
159}
160
161/// Common super class of ArrayType, StructType and VectorType.
162class CompositeType : public Type {
163protected:
164 explicit CompositeType(LLVMContext &C, TypeID tid) : Type(C, tid) {}
165
166public:
167 /// Given an index value into the type, return the type of the element.
168 Type *getTypeAtIndex(const Value *V) const;
169 Type *getTypeAtIndex(unsigned Idx) const;
170 bool indexValid(const Value *V) const;
171 bool indexValid(unsigned Idx) const;
172
173 /// Methods for support type inquiry through isa, cast, and dyn_cast.
174 static bool classof(const Type *T) {
175 return T->getTypeID() == ArrayTyID ||
176 T->getTypeID() == StructTyID ||
177 T->getTypeID() == VectorTyID;
178 }
179};
180
181/// Class to represent struct types. There are two different kinds of struct
182/// types: Literal structs and Identified structs.
183///
184/// Literal struct types (e.g. { i32, i32 }) are uniqued structurally, and must
185/// always have a body when created. You can get one of these by using one of
186/// the StructType::get() forms.
187///
188/// Identified structs (e.g. %foo or %42) may optionally have a name and are not
189/// uniqued. The names for identified structs are managed at the LLVMContext
190/// level, so there can only be a single identified struct with a given name in
191/// a particular LLVMContext. Identified structs may also optionally be opaque
192/// (have no body specified). You get one of these by using one of the
193/// StructType::create() forms.
194///
195/// Independent of what kind of struct you have, the body of a struct type are
196/// laid out in memory consequtively with the elements directly one after the
197/// other (if the struct is packed) or (if not packed) with padding between the
198/// elements as defined by DataLayout (which is required to match what the code
199/// generator for a target expects).
200///
201class StructType : public CompositeType {
202 StructType(LLVMContext &C) : CompositeType(C, StructTyID) {}
203
204 enum {
205 /// This is the contents of the SubClassData field.
206 SCDB_HasBody = 1,
207 SCDB_Packed = 2,
208 SCDB_IsLiteral = 4,
209 SCDB_IsSized = 8
210 };
211
212 /// For a named struct that actually has a name, this is a pointer to the
213 /// symbol table entry (maintained by LLVMContext) for the struct.
214 /// This is null if the type is an literal struct or if it is a identified
215 /// type that has an empty name.
216 void *SymbolTableEntry = nullptr;
217
218public:
219 StructType(const StructType &) = delete;
220 StructType &operator=(const StructType &) = delete;
221
222 /// This creates an identified struct.
223 static StructType *create(LLVMContext &Context, StringRef Name);
224 static StructType *create(LLVMContext &Context);
225
226 static StructType *create(ArrayRef<Type *> Elements, StringRef Name,
227 bool isPacked = false);
228 static StructType *create(ArrayRef<Type *> Elements);
229 static StructType *create(LLVMContext &Context, ArrayRef<Type *> Elements,
230 StringRef Name, bool isPacked = false);
231 static StructType *create(LLVMContext &Context, ArrayRef<Type *> Elements);
232 template <class... Tys>
233 static typename std::enable_if<are_base_of<Type, Tys...>::value,
234 StructType *>::type
235 create(StringRef Name, Type *elt1, Tys *... elts) {
236 assert(elt1 && "Cannot create a struct type with no elements with this")((elt1 && "Cannot create a struct type with no elements with this"
) ? static_cast<void> (0) : __assert_fail ("elt1 && \"Cannot create a struct type with no elements with this\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h"
, 236, __PRETTY_FUNCTION__));
237 SmallVector<llvm::Type *, 8> StructFields({elt1, elts...});
238 return create(StructFields, Name);
239 }
240
241 /// This static method is the primary way to create a literal StructType.
242 static StructType *get(LLVMContext &Context, ArrayRef<Type*> Elements,
243 bool isPacked = false);
244
245 /// Create an empty structure type.
246 static StructType *get(LLVMContext &Context, bool isPacked = false);
247
248 /// This static method is a convenience method for creating structure types by
249 /// specifying the elements as arguments. Note that this method always returns
250 /// a non-packed struct, and requires at least one element type.
251 template <class... Tys>
252 static typename std::enable_if<are_base_of<Type, Tys...>::value,
253 StructType *>::type
254 get(Type *elt1, Tys *... elts) {
255 assert(elt1 && "Cannot create a struct type with no elements with this")((elt1 && "Cannot create a struct type with no elements with this"
) ? static_cast<void> (0) : __assert_fail ("elt1 && \"Cannot create a struct type with no elements with this\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h"
, 255, __PRETTY_FUNCTION__));
256 LLVMContext &Ctx = elt1->getContext();
257 SmallVector<llvm::Type *, 8> StructFields({elt1, elts...});
258 return llvm::StructType::get(Ctx, StructFields);
259 }
260
261 bool isPacked() const { return (getSubclassData() & SCDB_Packed) != 0; }
262
263 /// Return true if this type is uniqued by structural equivalence, false if it
264 /// is a struct definition.
265 bool isLiteral() const { return (getSubclassData() & SCDB_IsLiteral) != 0; }
266
267 /// Return true if this is a type with an identity that has no body specified
268 /// yet. These prints as 'opaque' in .ll files.
269 bool isOpaque() const { return (getSubclassData() & SCDB_HasBody) == 0; }
270
271 /// isSized - Return true if this is a sized type.
272 bool isSized(SmallPtrSetImpl<Type *> *Visited = nullptr) const;
273
274 /// Return true if this is a named struct that has a non-empty name.
275 bool hasName() const { return SymbolTableEntry != nullptr; }
276
277 /// Return the name for this struct type if it has an identity.
278 /// This may return an empty string for an unnamed struct type. Do not call
279 /// this on an literal type.
280 StringRef getName() const;
281
282 /// Change the name of this type to the specified name, or to a name with a
283 /// suffix if there is a collision. Do not call this on an literal type.
284 void setName(StringRef Name);
285
286 /// Specify a body for an opaque identified type.
287 void setBody(ArrayRef<Type*> Elements, bool isPacked = false);
288
289 template <typename... Tys>
290 typename std::enable_if<are_base_of<Type, Tys...>::value, void>::type
291 setBody(Type *elt1, Tys *... elts) {
292 assert(elt1 && "Cannot create a struct type with no elements with this")((elt1 && "Cannot create a struct type with no elements with this"
) ? static_cast<void> (0) : __assert_fail ("elt1 && \"Cannot create a struct type with no elements with this\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h"
, 292, __PRETTY_FUNCTION__));
293 SmallVector<llvm::Type *, 8> StructFields({elt1, elts...});
294 setBody(StructFields);
295 }
296
297 /// Return true if the specified type is valid as a element type.
298 static bool isValidElementType(Type *ElemTy);
299
300 // Iterator access to the elements.
301 using element_iterator = Type::subtype_iterator;
302
303 element_iterator element_begin() const { return ContainedTys; }
304 element_iterator element_end() const { return &ContainedTys[NumContainedTys];}
305 ArrayRef<Type *> const elements() const {
306 return makeArrayRef(element_begin(), element_end());
307 }
308
309 /// Return true if this is layout identical to the specified struct.
310 bool isLayoutIdentical(StructType *Other) const;
311
312 /// Random access to the elements
313 unsigned getNumElements() const { return NumContainedTys; }
314 Type *getElementType(unsigned N) const {
315 assert(N < NumContainedTys && "Element number out of range!")((N < NumContainedTys && "Element number out of range!"
) ? static_cast<void> (0) : __assert_fail ("N < NumContainedTys && \"Element number out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h"
, 315, __PRETTY_FUNCTION__));
316 return ContainedTys[N];
317 }
318
319 /// Methods for support type inquiry through isa, cast, and dyn_cast.
320 static bool classof(const Type *T) {
321 return T->getTypeID() == StructTyID;
322 }
323};
324
325StringRef Type::getStructName() const {
326 return cast<StructType>(this)->getName();
327}
328
329unsigned Type::getStructNumElements() const {
330 return cast<StructType>(this)->getNumElements();
331}
332
333Type *Type::getStructElementType(unsigned N) const {
334 return cast<StructType>(this)->getElementType(N);
335}
336
337/// This is the superclass of the array and vector type classes. Both of these
338/// represent "arrays" in memory. The array type represents a specifically sized
339/// array, and the vector type represents a specifically sized array that allows
340/// for use of SIMD instructions. SequentialType holds the common features of
341/// both, which stem from the fact that both lay their components out in memory
342/// identically.
343class SequentialType : public CompositeType {
344 Type *ContainedType; ///< Storage for the single contained type.
345 uint64_t NumElements;
346
347protected:
348 SequentialType(TypeID TID, Type *ElType, uint64_t NumElements)
349 : CompositeType(ElType->getContext(), TID), ContainedType(ElType),
350 NumElements(NumElements) {
351 ContainedTys = &ContainedType;
352 NumContainedTys = 1;
353 }
354
355public:
356 SequentialType(const SequentialType &) = delete;
357 SequentialType &operator=(const SequentialType &) = delete;
358
359 uint64_t getNumElements() const { return NumElements; }
360 Type *getElementType() const { return ContainedType; }
361
362 /// Methods for support type inquiry through isa, cast, and dyn_cast.
363 static bool classof(const Type *T) {
364 return T->getTypeID() == ArrayTyID || T->getTypeID() == VectorTyID;
365 }
366};
367
368/// Class to represent array types.
369class ArrayType : public SequentialType {
370 ArrayType(Type *ElType, uint64_t NumEl);
371
372public:
373 ArrayType(const ArrayType &) = delete;
374 ArrayType &operator=(const ArrayType &) = delete;
375
376 /// This static method is the primary way to construct an ArrayType
377 static ArrayType *get(Type *ElementType, uint64_t NumElements);
378
379 /// Return true if the specified type is valid as a element type.
380 static bool isValidElementType(Type *ElemTy);
381
382 /// Methods for support type inquiry through isa, cast, and dyn_cast.
383 static bool classof(const Type *T) {
384 return T->getTypeID() == ArrayTyID;
385 }
386};
387
388uint64_t Type::getArrayNumElements() const {
389 return cast<ArrayType>(this)->getNumElements();
390}
391
392/// Class to represent vector types.
393class VectorType : public SequentialType {
394 VectorType(Type *ElType, unsigned NumEl);
395
396public:
397 VectorType(const VectorType &) = delete;
398 VectorType &operator=(const VectorType &) = delete;
399
400 /// This static method is the primary way to construct an VectorType.
401 static VectorType *get(Type *ElementType, unsigned NumElements);
402
403 /// This static method gets a VectorType with the same number of elements as
404 /// the input type, and the element type is an integer type of the same width
405 /// as the input element type.
406 static VectorType *getInteger(VectorType *VTy) {
407 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
408 assert(EltBits && "Element size must be of a non-zero size")((EltBits && "Element size must be of a non-zero size"
) ? static_cast<void> (0) : __assert_fail ("EltBits && \"Element size must be of a non-zero size\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h"
, 408, __PRETTY_FUNCTION__));
409 Type *EltTy = IntegerType::get(VTy->getContext(), EltBits);
410 return VectorType::get(EltTy, VTy->getNumElements());
411 }
412
413 /// This static method is like getInteger except that the element types are
414 /// twice as wide as the elements in the input type.
415 static VectorType *getExtendedElementVectorType(VectorType *VTy) {
416 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
417 Type *EltTy = IntegerType::get(VTy->getContext(), EltBits * 2);
418 return VectorType::get(EltTy, VTy->getNumElements());
419 }
420
421 /// This static method is like getInteger except that the element types are
422 /// half as wide as the elements in the input type.
423 static VectorType *getTruncatedElementVectorType(VectorType *VTy) {
424 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
425 assert((EltBits & 1) == 0 &&(((EltBits & 1) == 0 && "Cannot truncate vector element with odd bit-width"
) ? static_cast<void> (0) : __assert_fail ("(EltBits & 1) == 0 && \"Cannot truncate vector element with odd bit-width\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h"
, 426, __PRETTY_FUNCTION__))
426 "Cannot truncate vector element with odd bit-width")(((EltBits & 1) == 0 && "Cannot truncate vector element with odd bit-width"
) ? static_cast<void> (0) : __assert_fail ("(EltBits & 1) == 0 && \"Cannot truncate vector element with odd bit-width\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h"
, 426, __PRETTY_FUNCTION__));
427 Type *EltTy = IntegerType::get(VTy->getContext(), EltBits / 2);
428 return VectorType::get(EltTy, VTy->getNumElements());
429 }
430
431 /// This static method returns a VectorType with half as many elements as the
432 /// input type and the same element type.
433 static VectorType *getHalfElementsVectorType(VectorType *VTy) {
434 unsigned NumElts = VTy->getNumElements();
435 assert ((NumElts & 1) == 0 &&(((NumElts & 1) == 0 && "Cannot halve vector with odd number of elements."
) ? static_cast<void> (0) : __assert_fail ("(NumElts & 1) == 0 && \"Cannot halve vector with odd number of elements.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h"
, 436, __PRETTY_FUNCTION__))
436 "Cannot halve vector with odd number of elements.")(((NumElts & 1) == 0 && "Cannot halve vector with odd number of elements."
) ? static_cast<void> (0) : __assert_fail ("(NumElts & 1) == 0 && \"Cannot halve vector with odd number of elements.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/IR/DerivedTypes.h"
, 436, __PRETTY_FUNCTION__));
437 return VectorType::get(VTy->getElementType(), NumElts/2);
438 }
439
440 /// This static method returns a VectorType with twice as many elements as the
441 /// input type and the same element type.
442 static VectorType *getDoubleElementsVectorType(VectorType *VTy) {
443 unsigned NumElts = VTy->getNumElements();
444 return VectorType::get(VTy->getElementType(), NumElts*2);
445 }
446
447 /// Return true if the specified type is valid as a element type.
448 static bool isValidElementType(Type *ElemTy);
449
450 /// Return the number of bits in the Vector type.
451 /// Returns zero when the vector is a vector of pointers.
452 unsigned getBitWidth() const {
453 return getNumElements() * getElementType()->getPrimitiveSizeInBits();
454 }
455
456 /// Methods for support type inquiry through isa, cast, and dyn_cast.
457 static bool classof(const Type *T) {
458 return T->getTypeID() == VectorTyID;
14
←
Calling 'Type::getTypeID'
→
15
←
Returning from 'Type::getTypeID'
→
16
←
Assuming the condition is true
→
459 }
460};
461
462unsigned Type::getVectorNumElements() const {
463 return cast<VectorType>(this)->getNumElements();
2
←
Calling 'cast'
→
24
←
Returning from 'cast'
→
25
←
Calling 'SequentialType::getNumElements'
→
26
←
Returning from 'SequentialType::getNumElements'
→
464}
465
466/// Class to represent pointers.
467class PointerType : public Type {
468 explicit PointerType(Type *ElType, unsigned AddrSpace);
469
470 Type *PointeeTy;
471
472public:
473 PointerType(const PointerType &) = delete;
474 PointerType &operator=(const PointerType &) = delete;
475
476 /// This constructs a pointer to an object of the specified type in a numbered
477 /// address space.
478 static PointerType *get(Type *ElementType, unsigned AddressSpace);
479
480 /// This constructs a pointer to an object of the specified type in the
481 /// generic address space (address space zero).
482 static PointerType *getUnqual(Type *ElementType) {
483 return PointerType::get(ElementType, 0);
484 }
485
486 Type *getElementType() const { return PointeeTy; }
487
488 /// Return true if the specified type is valid as a element type.
489 static bool isValidElementType(Type *ElemTy);
490
491 /// Return true if we can load or store from a pointer to this type.
492 static bool isLoadableOrStorableType(Type *ElemTy);
493
494 /// Return the address space of the Pointer type.
495 inline unsigned getAddressSpace() const { return getSubclassData(); }
496
497 /// Implement support type inquiry through isa, cast, and dyn_cast.
498 static bool classof(const Type *T) {
499 return T->getTypeID() == PointerTyID;
500 }
501};
502
503unsigned Type::getPointerAddressSpace() const {
504 return cast<PointerType>(getScalarType())->getAddressSpace();
505}
506
507} // end namespace llvm
508
509#endif // LLVM_IR_DERIVEDTYPES_H
←

/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h

1//===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), cast_or_null<X>(),
11// and dyn_cast_or_null<X>() templates.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_SUPPORT_CASTING_H
16#define LLVM_SUPPORT_CASTING_H
17
18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/type_traits.h"
20#include <cassert>
21#include <memory>
22#include <type_traits>
23
24namespace llvm {
25
26//===----------------------------------------------------------------------===//
27// isa<x> Support Templates
28//===----------------------------------------------------------------------===//
29
30// Define a template that can be specialized by smart pointers to reflect the
31// fact that they are automatically dereferenced, and are not involved with the
32// template selection process... the default implementation is a noop.
33//
34template<typename From> struct simplify_type {
35 using SimpleType = From; // The real type this represents...
36
37 // An accessor to get the real value...
38 static SimpleType &getSimplifiedValue(From &Val) { return Val; }
39};
40
41template<typename From> struct simplify_type<const From> {
42 using NonConstSimpleType = typename simplify_type<From>::SimpleType;
43 using SimpleType =
44 typename add_const_past_pointer<NonConstSimpleType>::type;
45 using RetType =
46 typename add_lvalue_reference_if_not_pointer<SimpleType>::type;
47
48 static RetType getSimplifiedValue(const From& Val) {
49 return simplify_type<From>::getSimplifiedValue(const_cast<From&>(Val));
6
←
Calling 'simplify_type::getSimplifiedValue'
→
7
←
Returning from 'simplify_type::getSimplifiedValue'
→
50 }
51};
52
53// The core of the implementation of isa<X> is here; To and From should be
54// the names of classes. This template can be specialized to customize the
55// implementation of isa<> without rewriting it from scratch.
56template <typename To, typename From, typename Enabler = void>
57struct isa_impl {
58 static inline bool doit(const From &Val) {
59 return To::classof(&Val);
13
←
Calling 'VectorType::classof'
→
17
←
Returning from 'VectorType::classof'
→
60 }
61};
62
63/// \brief Always allow upcasts, and perform no dynamic check for them.
64template <typename To, typename From>
65struct isa_impl<
66 To, From, typename std::enable_if<std::is_base_of<To, From>::value>::type> {
67 static inline bool doit(const From &) { return true; }
68};
69
70template <typename To, typename From> struct isa_impl_cl {
71 static inline bool doit(const From &Val) {
72 return isa_impl<To, From>::doit(Val);
73 }
74};
75
76template <typename To, typename From> struct isa_impl_cl<To, const From> {
77 static inline bool doit(const From &Val) {
78 return isa_impl<To, From>::doit(Val);
79 }
80};
81
82template <typename To, typename From>
83struct isa_impl_cl<To, const std::unique_ptr<From>> {
84 static inline bool doit(const std::unique_ptr<From> &Val) {
85 assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 85, __PRETTY_FUNCTION__));
86 return isa_impl_cl<To, From>::doit(*Val);
87 }
88};
89
90template <typename To, typename From> struct isa_impl_cl<To, From*> {
91 static inline bool doit(const From *Val) {
92 assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 92, __PRETTY_FUNCTION__));
93 return isa_impl<To, From>::doit(*Val);
94 }
95};
96
97template <typename To, typename From> struct isa_impl_cl<To, From*const> {
98 static inline bool doit(const From *Val) {
99 assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 99, __PRETTY_FUNCTION__));
100 return isa_impl<To, From>::doit(*Val);
101 }
102};
103
104template <typename To, typename From> struct isa_impl_cl<To, const From*> {
105 static inline bool doit(const From *Val) {
106 assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 106, __PRETTY_FUNCTION__));
11
←
Within the expansion of the macro 'assert':
→
107 return isa_impl<To, From>::doit(*Val);
12
←
Calling 'isa_impl::doit'
→
18
←
Returning from 'isa_impl::doit'
→
108 }
109};
110
111template <typename To, typename From> struct isa_impl_cl<To, const From*const> {
112 static inline bool doit(const From *Val) {
113 assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 113, __PRETTY_FUNCTION__));
114 return isa_impl<To, From>::doit(*Val);
115 }
116};
117
118template<typename To, typename From, typename SimpleFrom>
119struct isa_impl_wrap {
120 // When From != SimplifiedType, we can simplify the type some more by using
121 // the simplify_type template.
122 static bool doit(const From &Val) {
123 return isa_impl_wrap<To, SimpleFrom,
9
←
Calling 'isa_impl_wrap::doit'
→
20
←
Returning from 'isa_impl_wrap::doit'
→
124 typename simplify_type<SimpleFrom>::SimpleType>::doit(
125 simplify_type<const From>::getSimplifiedValue(Val));
5
←
Calling 'simplify_type::getSimplifiedValue'
→
8
←
Returning from 'simplify_type::getSimplifiedValue'
→
126 }
127};
128
129template<typename To, typename FromTy>
130struct isa_impl_wrap<To, FromTy, FromTy> {
131 // When From == SimpleType, we are as simple as we are going to get.
132 static bool doit(const FromTy &Val) {
133 return isa_impl_cl<To,FromTy>::doit(Val);
10
←
Calling 'isa_impl_cl::doit'
→
19
←
Returning from 'isa_impl_cl::doit'
→
134 }
135};
136
137// isa<X> - Return true if the parameter to the template is an instance of the
138// template type argument. Used like this:
139//
140// if (isa<Type>(myVal)) { ... }
141//
142template <class X, class Y> LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
143 return isa_impl_wrap<X, const Y,
4
←
Calling 'isa_impl_wrap::doit'
→
21
←
Returning from 'isa_impl_wrap::doit'
→
144 typename simplify_type<const Y>::SimpleType>::doit(Val);
145}
146
147//===----------------------------------------------------------------------===//
148// cast<x> Support Templates
149//===----------------------------------------------------------------------===//
150
151template<class To, class From> struct cast_retty;
152
153// Calculate what type the 'cast' function should return, based on a requested
154// type of To and a source type of From.
155template<class To, class From> struct cast_retty_impl {
156 using ret_type = To &; // Normal case, return Ty&
157};
158template<class To, class From> struct cast_retty_impl<To, const From> {
159 using ret_type = const To &; // Normal case, return Ty&
160};
161
162template<class To, class From> struct cast_retty_impl<To, From*> {
163 using ret_type = To *; // Pointer arg case, return Ty*
164};
165
166template<class To, class From> struct cast_retty_impl<To, const From*> {
167 using ret_type = const To *; // Constant pointer arg case, return const Ty*
168};
169
170template<class To, class From> struct cast_retty_impl<To, const From*const> {
171 using ret_type = const To *; // Constant pointer arg case, return const Ty*
172};
173
174template <class To, class From>
175struct cast_retty_impl<To, std::unique_ptr<From>> {
176private:
177 using PointerType = typename cast_retty_impl<To, From *>::ret_type;
178 using ResultType = typename std::remove_pointer<PointerType>::type;
179
180public:
181 using ret_type = std::unique_ptr<ResultType>;
182};
183
184template<class To, class From, class SimpleFrom>
185struct cast_retty_wrap {
186 // When the simplified type and the from type are not the same, use the type
187 // simplifier to reduce the type, then reuse cast_retty_impl to get the
188 // resultant type.
189 using ret_type = typename cast_retty<To, SimpleFrom>::ret_type;
190};
191
192template<class To, class FromTy>
193struct cast_retty_wrap<To, FromTy, FromTy> {
194 // When the simplified type is equal to the from type, use it directly.
195 using ret_type = typename cast_retty_impl<To,FromTy>::ret_type;
196};
197
198template<class To, class From>
199struct cast_retty {
200 using ret_type = typename cast_retty_wrap<
201 To, From, typename simplify_type<From>::SimpleType>::ret_type;
202};
203
204// Ensure the non-simple values are converted using the simplify_type template
205// that may be specialized by smart pointers...
206//
207template<class To, class From, class SimpleFrom> struct cast_convert_val {
208 // This is not a simple type, use the template to simplify it...
209 static typename cast_retty<To, From>::ret_type doit(From &Val) {
210 return cast_convert_val<To, SimpleFrom,
211 typename simplify_type<SimpleFrom>::SimpleType>::doit(
212 simplify_type<From>::getSimplifiedValue(Val));
213 }
214};
215
216template<class To, class FromTy> struct cast_convert_val<To,FromTy,FromTy> {
217 // This _is_ a simple type, just cast it.
218 static typename cast_retty<To, FromTy>::ret_type doit(const FromTy &Val) {
219 typename cast_retty<To, FromTy>::ret_type Res2
220 = (typename cast_retty<To, FromTy>::ret_type)const_cast<FromTy&>(Val);
221 return Res2;
222 }
223};
224
225template <class X> struct is_simple_type {
226 static const bool value =
227 std::is_same<X, typename simplify_type<X>::SimpleType>::value;
228};
229
230// cast<X> - Return the argument parameter cast to the specified type. This
231// casting operator asserts that the type is correct, so it does not return null
232// on failure. It does not allow a null argument (use cast_or_null for that).
233// It is typically used like this:
234//
235// cast<Instruction>(myVal)->getParent()
236//
237template <class X, class Y>
238inline typename std::enable_if<!is_simple_type<Y>::value,
239 typename cast_retty<X, const Y>::ret_type>::type
240cast(const Y &Val) {
241 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 241, __PRETTY_FUNCTION__));
242 return cast_convert_val<
243 X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
244}
245
246template <class X, class Y>
247inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
248 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 248, __PRETTY_FUNCTION__));
249 return cast_convert_val<X, Y,
250 typename simplify_type<Y>::SimpleType>::doit(Val);
251}
252
253template <class X, class Y>
254inline typename cast_retty<X, Y *>::ret_type cast(Y *Val) {
255 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 255, __PRETTY_FUNCTION__));
3
←
Within the expansion of the macro 'assert':
→
a
Calling 'isa'
b
Returning from 'isa'
256 return cast_convert_val<X, Y*,
22
←
Calling 'cast_convert_val::doit'
→
23
←
Returning from 'cast_convert_val::doit'
→
257 typename simplify_type<Y*>::SimpleType>::doit(Val);
258}
259
260template <class X, class Y>
261inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
262cast(std::unique_ptr<Y> &&Val) {
263 assert(isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!")((isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val.get()) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 263, __PRETTY_FUNCTION__));
264 using ret_type = typename cast_retty<X, std::unique_ptr<Y>>::ret_type;
265 return ret_type(
266 cast_convert_val<X, Y *, typename simplify_type<Y *>::SimpleType>::doit(
267 Val.release()));
268}
269
270// cast_or_null<X> - Functionally identical to cast, except that a null value is
271// accepted.
272//
273template <class X, class Y>
274LLVM_NODISCARD[[clang::warn_unused_result]] inline
275 typename std::enable_if<!is_simple_type<Y>::value,
276 typename cast_retty<X, const Y>::ret_type>::type
277 cast_or_null(const Y &Val) {
278 if (!Val)
279 return nullptr;
280 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 280, __PRETTY_FUNCTION__));
281 return cast<X>(Val);
282}
283
284template <class X, class Y>
285LLVM_NODISCARD[[clang::warn_unused_result]] inline
286 typename std::enable_if<!is_simple_type<Y>::value,
287 typename cast_retty<X, Y>::ret_type>::type
288 cast_or_null(Y &Val) {
289 if (!Val)
290 return nullptr;
291 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 291, __PRETTY_FUNCTION__));
292 return cast<X>(Val);
293}
294
295template <class X, class Y>
296LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
297cast_or_null(Y *Val) {
298 if (!Val) return nullptr;
299 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318001/include/llvm/Support/Casting.h"
, 299, __PRETTY_FUNCTION__));
300 return cast<X>(Val);
301}
302
303template <class X, class Y>
304inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
305cast_or_null(std::unique_ptr<Y> &&Val) {
306 if (!Val)
307 return nullptr;
308 return cast<X>(std::move(Val));
309}
310
311// dyn_cast<X> - Return the argument parameter cast to the specified type. This
312// casting operator returns null if the argument is of the wrong type, so it can
313// be used to test for a type as well as cast if successful. This should be
314// used in the context of an if statement like this:
315//
316// if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... }
317//
318
319template <class X, class Y>
320LLVM_NODISCARD[[clang::warn_unused_result]] inline
321 typename std::enable_if<!is_simple_type<Y>::value,
322 typename cast_retty<X, const Y>::ret_type>::type
323 dyn_cast(const Y &Val) {
324 return isa<X>(Val) ? cast<X>(Val) : nullptr;
325}
326
327template <class X, class Y>
328LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
329 return isa<X>(Val) ? cast<X>(Val) : nullptr;
330}
331
332template <class X, class Y>
333LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) {
334 return isa<X>(Val) ? cast<X>(Val) : nullptr;
335}
336
337// dyn_cast_or_null<X> - Functionally identical to dyn_cast, except that a null
338// value is accepted.
339//
340template <class X, class Y>
341LLVM_NODISCARD[[clang::warn_unused_result]] inline
342 typename std::enable_if<!is_simple_type<Y>::value,
343 typename cast_retty<X, const Y>::ret_type>::type
344 dyn_cast_or_null(const Y &Val) {
345 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
346}
347
348template <class X, class Y>
349LLVM_NODISCARD[[clang::warn_unused_result]] inline
350 typename std::enable_if<!is_simple_type<Y>::value,
351 typename cast_retty<X, Y>::ret_type>::type
352 dyn_cast_or_null(Y &Val) {
353 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
354}
355
356template <class X, class Y>
357LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
358dyn_cast_or_null(Y *Val) {
359 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
360}
361
362// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>,
363// taking ownership of the input pointer iff isa<X>(Val) is true. If the
364// cast is successful, From refers to nullptr on exit and the casted value
365// is returned. If the cast is unsuccessful, the function returns nullptr
366// and From is unchanged.
367template <class X, class Y>
368LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &Val)
369 -> decltype(cast<X>(Val)) {
370 if (!isa<X>(Val))
371 return nullptr;
372 return cast<X>(std::move(Val));
373}
374
375template <class X, class Y>
376LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val)
377 -> decltype(cast<X>(Val)) {
378 return unique_dyn_cast<X, Y>(Val);
379}
380
381// dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, except that
382// a null value is accepted.
383template <class X, class Y>
384LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val)
385 -> decltype(cast<X>(Val)) {
386 if (!Val)
387 return nullptr;
388 return unique_dyn_cast<X, Y>(Val);
389}
390
391template <class X, class Y>
392LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val)
393 -> decltype(cast<X>(Val)) {
394 return unique_dyn_cast_or_null<X, Y>(Val);
395}
396
397} // end namespace llvm
398
399#endif // LLVM_SUPPORT_CASTING_H