Bug Summary

File:tools/clang/lib/CodeGen/CGBuiltin.cpp
Warning:line 7453, column 22
Division by zero

Annotated Source Code

/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp

1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This contains code to emit Builtin calls as LLVM code.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCXXABI.h"
15#include "CGObjCRuntime.h"
16#include "CGOpenCLRuntime.h"
17#include "CodeGenFunction.h"
18#include "CodeGenModule.h"
19#include "ConstantEmitter.h"
20#include "TargetInfo.h"
21#include "clang/AST/ASTContext.h"
22#include "clang/AST/Decl.h"
23#include "clang/Analysis/Analyses/OSLog.h"
24#include "clang/Basic/TargetBuiltins.h"
25#include "clang/Basic/TargetInfo.h"
26#include "clang/CodeGen/CGFunctionInfo.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/IR/CallSite.h"
29#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/InlineAsm.h"
31#include "llvm/IR/Intrinsics.h"
32#include "llvm/IR/MDBuilder.h"
33#include "llvm/Support/ConvertUTF.h"
34#include "llvm/Support/ScopedPrinter.h"
35#include "llvm/Support/TargetParser.h"
36#include <sstream>
37
38using namespace clang;
39using namespace CodeGen;
40using namespace llvm;
41
42static
43int64_t clamp(int64_t Value, int64_t Low, int64_t High) {
44 return std::min(High, std::max(Low, Value));
45}
46
47/// getBuiltinLibFunction - Given a builtin id for a function like
48/// "__builtin_fabsf", return a Function* for "fabsf".
49llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
50 unsigned BuiltinID) {
51 assert(Context.BuiltinInfo.isLibFunction(BuiltinID))(static_cast <bool> (Context.BuiltinInfo.isLibFunction(
BuiltinID)) ? void (0) : __assert_fail ("Context.BuiltinInfo.isLibFunction(BuiltinID)"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 51, __extension__ __PRETTY_FUNCTION__))
;
52
53 // Get the name, skip over the __builtin_ prefix (if necessary).
54 StringRef Name;
55 GlobalDecl D(FD);
56
57 // If the builtin has been declared explicitly with an assembler label,
58 // use the mangled name. This differs from the plain label on platforms
59 // that prefix labels.
60 if (FD->hasAttr<AsmLabelAttr>())
61 Name = getMangledName(D);
62 else
63 Name = Context.BuiltinInfo.getName(BuiltinID) + 10;
64
65 llvm::FunctionType *Ty =
66 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
67
68 return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
69}
70
71/// Emit the conversions required to turn the given value into an
72/// integer of the given size.
73static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
74 QualType T, llvm::IntegerType *IntType) {
75 V = CGF.EmitToMemory(V, T);
76
77 if (V->getType()->isPointerTy())
78 return CGF.Builder.CreatePtrToInt(V, IntType);
79
80 assert(V->getType() == IntType)(static_cast <bool> (V->getType() == IntType) ? void
(0) : __assert_fail ("V->getType() == IntType", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 80, __extension__ __PRETTY_FUNCTION__))
;
81 return V;
82}
83
84static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
85 QualType T, llvm::Type *ResultType) {
86 V = CGF.EmitFromMemory(V, T);
87
88 if (ResultType->isPointerTy())
89 return CGF.Builder.CreateIntToPtr(V, ResultType);
90
91 assert(V->getType() == ResultType)(static_cast <bool> (V->getType() == ResultType) ? void
(0) : __assert_fail ("V->getType() == ResultType", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 91, __extension__ __PRETTY_FUNCTION__))
;
92 return V;
93}
94
95/// Utility to insert an atomic instruction based on Instrinsic::ID
96/// and the expression node.
97static Value *MakeBinaryAtomicValue(CodeGenFunction &CGF,
98 llvm::AtomicRMWInst::BinOp Kind,
99 const CallExpr *E) {
100 QualType T = E->getType();
101 assert(E->getArg(0)->getType()->isPointerType())(static_cast <bool> (E->getArg(0)->getType()->
isPointerType()) ? void (0) : __assert_fail ("E->getArg(0)->getType()->isPointerType()"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 101, __extension__ __PRETTY_FUNCTION__))
;
102 assert(CGF.getContext().hasSameUnqualifiedType(T,(static_cast <bool> (CGF.getContext().hasSameUnqualifiedType
(T, E->getArg(0)->getType()->getPointeeType())) ? void
(0) : __assert_fail ("CGF.getContext().hasSameUnqualifiedType(T, E->getArg(0)->getType()->getPointeeType())"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 103, __extension__ __PRETTY_FUNCTION__))
103 E->getArg(0)->getType()->getPointeeType()))(static_cast <bool> (CGF.getContext().hasSameUnqualifiedType
(T, E->getArg(0)->getType()->getPointeeType())) ? void
(0) : __assert_fail ("CGF.getContext().hasSameUnqualifiedType(T, E->getArg(0)->getType()->getPointeeType())"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 103, __extension__ __PRETTY_FUNCTION__))
;
104 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()))(static_cast <bool> (CGF.getContext().hasSameUnqualifiedType
(T, E->getArg(1)->getType())) ? void (0) : __assert_fail
("CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 104, __extension__ __PRETTY_FUNCTION__))
;
105
106 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
107 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
108
109 llvm::IntegerType *IntType =
110 llvm::IntegerType::get(CGF.getLLVMContext(),
111 CGF.getContext().getTypeSize(T));
112 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
113
114 llvm::Value *Args[2];
115 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
116 Args[1] = CGF.EmitScalarExpr(E->getArg(1));
117 llvm::Type *ValueType = Args[1]->getType();
118 Args[1] = EmitToInt(CGF, Args[1], T, IntType);
119
120 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
121 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
122 return EmitFromInt(CGF, Result, T, ValueType);
123}
124
125static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
126 Value *Val = CGF.EmitScalarExpr(E->getArg(0));
127 Value *Address = CGF.EmitScalarExpr(E->getArg(1));
128
129 // Convert the type of the pointer to a pointer to the stored type.
130 Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
131 Value *BC = CGF.Builder.CreateBitCast(
132 Address, llvm::PointerType::getUnqual(Val->getType()), "cast");
133 LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType());
134 LV.setNontemporal(true);
135 CGF.EmitStoreOfScalar(Val, LV, false);
136 return nullptr;
137}
138
139static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
140 Value *Address = CGF.EmitScalarExpr(E->getArg(0));
141
142 LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType());
143 LV.setNontemporal(true);
144 return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
145}
146
147static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
148 llvm::AtomicRMWInst::BinOp Kind,
149 const CallExpr *E) {
150 return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
151}
152
153/// Utility to insert an atomic instruction based Instrinsic::ID and
154/// the expression node, where the return value is the result of the
155/// operation.
156static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
157 llvm::AtomicRMWInst::BinOp Kind,
158 const CallExpr *E,
159 Instruction::BinaryOps Op,
160 bool Invert = false) {
161 QualType T = E->getType();
162 assert(E->getArg(0)->getType()->isPointerType())(static_cast <bool> (E->getArg(0)->getType()->
isPointerType()) ? void (0) : __assert_fail ("E->getArg(0)->getType()->isPointerType()"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 162, __extension__ __PRETTY_FUNCTION__))
;
163 assert(CGF.getContext().hasSameUnqualifiedType(T,(static_cast <bool> (CGF.getContext().hasSameUnqualifiedType
(T, E->getArg(0)->getType()->getPointeeType())) ? void
(0) : __assert_fail ("CGF.getContext().hasSameUnqualifiedType(T, E->getArg(0)->getType()->getPointeeType())"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 164, __extension__ __PRETTY_FUNCTION__))
164 E->getArg(0)->getType()->getPointeeType()))(static_cast <bool> (CGF.getContext().hasSameUnqualifiedType
(T, E->getArg(0)->getType()->getPointeeType())) ? void
(0) : __assert_fail ("CGF.getContext().hasSameUnqualifiedType(T, E->getArg(0)->getType()->getPointeeType())"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 164, __extension__ __PRETTY_FUNCTION__))
;
165 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()))(static_cast <bool> (CGF.getContext().hasSameUnqualifiedType
(T, E->getArg(1)->getType())) ? void (0) : __assert_fail
("CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 165, __extension__ __PRETTY_FUNCTION__))
;
166
167 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
168 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
169
170 llvm::IntegerType *IntType =
171 llvm::IntegerType::get(CGF.getLLVMContext(),
172 CGF.getContext().getTypeSize(T));
173 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
174
175 llvm::Value *Args[2];
176 Args[1] = CGF.EmitScalarExpr(E->getArg(1));
177 llvm::Type *ValueType = Args[1]->getType();
178 Args[1] = EmitToInt(CGF, Args[1], T, IntType);
179 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
180
181 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
182 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
183 Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
184 if (Invert)
185 Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
186 llvm::ConstantInt::get(IntType, -1));
187 Result = EmitFromInt(CGF, Result, T, ValueType);
188 return RValue::get(Result);
189}
190
191/// @brief Utility to insert an atomic cmpxchg instruction.
192///
193/// @param CGF The current codegen function.
194/// @param E Builtin call expression to convert to cmpxchg.
195/// arg0 - address to operate on
196/// arg1 - value to compare with
197/// arg2 - new value
198/// @param ReturnBool Specifies whether to return success flag of
199/// cmpxchg result or the old value.
200///
201/// @returns result of cmpxchg, according to ReturnBool
202static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
203 bool ReturnBool) {
204 QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
205 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
206 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
207
208 llvm::IntegerType *IntType = llvm::IntegerType::get(
209 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
210 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
211
212 Value *Args[3];
213 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
214 Args[1] = CGF.EmitScalarExpr(E->getArg(1));
215 llvm::Type *ValueType = Args[1]->getType();
216 Args[1] = EmitToInt(CGF, Args[1], T, IntType);
217 Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
218
219 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
220 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
221 llvm::AtomicOrdering::SequentiallyConsistent);
222 if (ReturnBool)
223 // Extract boolean success flag and zext it to int.
224 return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
225 CGF.ConvertType(E->getType()));
226 else
227 // Extract old value and emit it using the same type as compare value.
228 return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
229 ValueType);
230}
231
232// Emit a simple mangled intrinsic that has 1 argument and a return type
233// matching the argument type.
234static Value *emitUnaryBuiltin(CodeGenFunction &CGF,
235 const CallExpr *E,
236 unsigned IntrinsicID) {
237 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
238
239 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
240 return CGF.Builder.CreateCall(F, Src0);
241}
242
243// Emit an intrinsic that has 2 operands of the same type as its result.
244static Value *emitBinaryBuiltin(CodeGenFunction &CGF,
245 const CallExpr *E,
246 unsigned IntrinsicID) {
247 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
248 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
249
250 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
251 return CGF.Builder.CreateCall(F, { Src0, Src1 });
252}
253
254// Emit an intrinsic that has 3 operands of the same type as its result.
255static Value *emitTernaryBuiltin(CodeGenFunction &CGF,
256 const CallExpr *E,
257 unsigned IntrinsicID) {
258 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
259 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
260 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
261
262 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
263 return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
264}
265
266// Emit an intrinsic that has 1 float or double operand, and 1 integer.
267static Value *emitFPIntBuiltin(CodeGenFunction &CGF,
268 const CallExpr *E,
269 unsigned IntrinsicID) {
270 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
271 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
272
273 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
274 return CGF.Builder.CreateCall(F, {Src0, Src1});
275}
276
277/// EmitFAbs - Emit a call to @llvm.fabs().
278static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
279 Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
280 llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
281 Call->setDoesNotAccessMemory();
282 return Call;
283}
284
285/// Emit the computation of the sign bit for a floating point value. Returns
286/// the i1 sign bit value.
287static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
288 LLVMContext &C = CGF.CGM.getLLVMContext();
289
290 llvm::Type *Ty = V->getType();
291 int Width = Ty->getPrimitiveSizeInBits();
292 llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
293 V = CGF.Builder.CreateBitCast(V, IntTy);
294 if (Ty->isPPC_FP128Ty()) {
295 // We want the sign bit of the higher-order double. The bitcast we just
296 // did works as if the double-double was stored to memory and then
297 // read as an i128. The "store" will put the higher-order double in the
298 // lower address in both little- and big-Endian modes, but the "load"
299 // will treat those bits as a different part of the i128: the low bits in
300 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
301 // we need to shift the high bits down to the low before truncating.
302 Width >>= 1;
303 if (CGF.getTarget().isBigEndian()) {
304 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
305 V = CGF.Builder.CreateLShr(V, ShiftCst);
306 }
307 // We are truncating value in order to extract the higher-order
308 // double, which we will be using to extract the sign from.
309 IntTy = llvm::IntegerType::get(C, Width);
310 V = CGF.Builder.CreateTrunc(V, IntTy);
311 }
312 Value *Zero = llvm::Constant::getNullValue(IntTy);
313 return CGF.Builder.CreateICmpSLT(V, Zero);
314}
315
316static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
317 const CallExpr *E, llvm::Constant *calleeValue) {
318 CGCallee callee = CGCallee::forDirect(calleeValue, FD);
319 return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
320}
321
322/// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
323/// depending on IntrinsicID.
324///
325/// \arg CGF The current codegen function.
326/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
327/// \arg X The first argument to the llvm.*.with.overflow.*.
328/// \arg Y The second argument to the llvm.*.with.overflow.*.
329/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
330/// \returns The result (i.e. sum/product) returned by the intrinsic.
331static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
332 const llvm::Intrinsic::ID IntrinsicID,
333 llvm::Value *X, llvm::Value *Y,
334 llvm::Value *&Carry) {
335 // Make sure we have integers of the same width.
336 assert(X->getType() == Y->getType() &&(static_cast <bool> (X->getType() == Y->getType()
&& "Arguments must be the same type. (Did you forget to make sure both "
"arguments have the same integer width?)") ? void (0) : __assert_fail
("X->getType() == Y->getType() && \"Arguments must be the same type. (Did you forget to make sure both \" \"arguments have the same integer width?)\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 338, __extension__ __PRETTY_FUNCTION__))
337 "Arguments must be the same type. (Did you forget to make sure both "(static_cast <bool> (X->getType() == Y->getType()
&& "Arguments must be the same type. (Did you forget to make sure both "
"arguments have the same integer width?)") ? void (0) : __assert_fail
("X->getType() == Y->getType() && \"Arguments must be the same type. (Did you forget to make sure both \" \"arguments have the same integer width?)\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 338, __extension__ __PRETTY_FUNCTION__))
338 "arguments have the same integer width?)")(static_cast <bool> (X->getType() == Y->getType()
&& "Arguments must be the same type. (Did you forget to make sure both "
"arguments have the same integer width?)") ? void (0) : __assert_fail
("X->getType() == Y->getType() && \"Arguments must be the same type. (Did you forget to make sure both \" \"arguments have the same integer width?)\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 338, __extension__ __PRETTY_FUNCTION__))
;
339
340 llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
341 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
342 Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
343 return CGF.Builder.CreateExtractValue(Tmp, 0);
344}
345
346static Value *emitRangedBuiltin(CodeGenFunction &CGF,
347 unsigned IntrinsicID,
348 int low, int high) {
349 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
350 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
351 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
352 llvm::Instruction *Call = CGF.Builder.CreateCall(F);
353 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
354 return Call;
355}
356
357namespace {
358 struct WidthAndSignedness {
359 unsigned Width;
360 bool Signed;
361 };
362}
363
364static WidthAndSignedness
365getIntegerWidthAndSignedness(const clang::ASTContext &context,
366 const clang::QualType Type) {
367 assert(Type->isIntegerType() && "Given type is not an integer.")(static_cast <bool> (Type->isIntegerType() &&
"Given type is not an integer.") ? void (0) : __assert_fail (
"Type->isIntegerType() && \"Given type is not an integer.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 367, __extension__ __PRETTY_FUNCTION__))
;
368 unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width;
369 bool Signed = Type->isSignedIntegerType();
370 return {Width, Signed};
371}
372
373// Given one or more integer types, this function produces an integer type that
374// encompasses them: any value in one of the given types could be expressed in
375// the encompassing type.
376static struct WidthAndSignedness
377EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
378 assert(Types.size() > 0 && "Empty list of types.")(static_cast <bool> (Types.size() > 0 && "Empty list of types."
) ? void (0) : __assert_fail ("Types.size() > 0 && \"Empty list of types.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 378, __extension__ __PRETTY_FUNCTION__))
;
379
380 // If any of the given types is signed, we must return a signed type.
381 bool Signed = false;
382 for (const auto &Type : Types) {
383 Signed |= Type.Signed;
384 }
385
386 // The encompassing type must have a width greater than or equal to the width
387 // of the specified types. Aditionally, if the encompassing type is signed,
388 // its width must be strictly greater than the width of any unsigned types
389 // given.
390 unsigned Width = 0;
391 for (const auto &Type : Types) {
392 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
393 if (Width < MinWidth) {
394 Width = MinWidth;
395 }
396 }
397
398 return {Width, Signed};
399}
400
401Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
402 llvm::Type *DestType = Int8PtrTy;
403 if (ArgValue->getType() != DestType)
404 ArgValue =
405 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
406
407 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
408 return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
409}
410
411/// Checks if using the result of __builtin_object_size(p, @p From) in place of
412/// __builtin_object_size(p, @p To) is correct
413static bool areBOSTypesCompatible(int From, int To) {
414 // Note: Our __builtin_object_size implementation currently treats Type=0 and
415 // Type=2 identically. Encoding this implementation detail here may make
416 // improving __builtin_object_size difficult in the future, so it's omitted.
417 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
418}
419
420static llvm::Value *
421getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
422 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
423}
424
425llvm::Value *
426CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
427 llvm::IntegerType *ResType,
428 llvm::Value *EmittedE) {
429 uint64_t ObjectSize;
430 if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
431 return emitBuiltinObjectSize(E, Type, ResType, EmittedE);
432 return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
433}
434
435/// Returns a Value corresponding to the size of the given expression.
436/// This Value may be either of the following:
437/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
438/// it)
439/// - A call to the @llvm.objectsize intrinsic
440///
441/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
442/// and we wouldn't otherwise try to reference a pass_object_size parameter,
443/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
444llvm::Value *
445CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
446 llvm::IntegerType *ResType,
447 llvm::Value *EmittedE) {
448 // We need to reference an argument if the pointer is a parameter with the
449 // pass_object_size attribute.
450 if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
451 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
452 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
453 if (Param != nullptr && PS != nullptr &&
454 areBOSTypesCompatible(PS->getType(), Type)) {
455 auto Iter = SizeArguments.find(Param);
456 assert(Iter != SizeArguments.end())(static_cast <bool> (Iter != SizeArguments.end()) ? void
(0) : __assert_fail ("Iter != SizeArguments.end()", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 456, __extension__ __PRETTY_FUNCTION__))
;
457
458 const ImplicitParamDecl *D = Iter->second;
459 auto DIter = LocalDeclMap.find(D);
460 assert(DIter != LocalDeclMap.end())(static_cast <bool> (DIter != LocalDeclMap.end()) ? void
(0) : __assert_fail ("DIter != LocalDeclMap.end()", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 460, __extension__ __PRETTY_FUNCTION__))
;
461
462 return EmitLoadOfScalar(DIter->second, /*volatile=*/false,
463 getContext().getSizeType(), E->getLocStart());
464 }
465 }
466
467 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
468 // evaluate E for side-effects. In either case, we shouldn't lower to
469 // @llvm.objectsize.
470 if (Type == 3 || (!EmittedE && E->HasSideEffects(getContext())))
471 return getDefaultBuiltinObjectSizeResult(Type, ResType);
472
473 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
474 assert(Ptr->getType()->isPointerTy() &&(static_cast <bool> (Ptr->getType()->isPointerTy(
) && "Non-pointer passed to __builtin_object_size?") ?
void (0) : __assert_fail ("Ptr->getType()->isPointerTy() && \"Non-pointer passed to __builtin_object_size?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 475, __extension__ __PRETTY_FUNCTION__))
475 "Non-pointer passed to __builtin_object_size?")(static_cast <bool> (Ptr->getType()->isPointerTy(
) && "Non-pointer passed to __builtin_object_size?") ?
void (0) : __assert_fail ("Ptr->getType()->isPointerTy() && \"Non-pointer passed to __builtin_object_size?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 475, __extension__ __PRETTY_FUNCTION__))
;
476
477 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
478
479 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
480 Value *Min = Builder.getInt1((Type & 2) != 0);
481 // For GCC compatability, __builtin_object_size treat NULL as unknown size.
482 Value *NullIsUnknown = Builder.getTrue();
483 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown});
484}
485
486// Many of MSVC builtins are on both x64 and ARM; to avoid repeating code, we
487// handle them here.
488enum class CodeGenFunction::MSVCIntrin {
489 _BitScanForward,
490 _BitScanReverse,
491 _InterlockedAnd,
492 _InterlockedDecrement,
493 _InterlockedExchange,
494 _InterlockedExchangeAdd,
495 _InterlockedExchangeSub,
496 _InterlockedIncrement,
497 _InterlockedOr,
498 _InterlockedXor,
499 _interlockedbittestandset,
500 __fastfail,
501};
502
503Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
504 const CallExpr *E) {
505 switch (BuiltinID) {
506 case MSVCIntrin::_BitScanForward:
507 case MSVCIntrin::_BitScanReverse: {
508 Value *ArgValue = EmitScalarExpr(E->getArg(1));
509
510 llvm::Type *ArgType = ArgValue->getType();
511 llvm::Type *IndexType =
512 EmitScalarExpr(E->getArg(0))->getType()->getPointerElementType();
513 llvm::Type *ResultType = ConvertType(E->getType());
514
515 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
516 Value *ResZero = llvm::Constant::getNullValue(ResultType);
517 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
518
519 BasicBlock *Begin = Builder.GetInsertBlock();
520 BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
521 Builder.SetInsertPoint(End);
522 PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
523
524 Builder.SetInsertPoint(Begin);
525 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
526 BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
527 Builder.CreateCondBr(IsZero, End, NotZero);
528 Result->addIncoming(ResZero, Begin);
529
530 Builder.SetInsertPoint(NotZero);
531 Address IndexAddress = EmitPointerWithAlignment(E->getArg(0));
532
533 if (BuiltinID == MSVCIntrin::_BitScanForward) {
534 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
535 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
536 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
537 Builder.CreateStore(ZeroCount, IndexAddress, false);
538 } else {
539 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
540 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
541
542 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
543 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
544 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
545 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
546 Builder.CreateStore(Index, IndexAddress, false);
547 }
548 Builder.CreateBr(End);
549 Result->addIncoming(ResOne, NotZero);
550
551 Builder.SetInsertPoint(End);
552 return Result;
553 }
554 case MSVCIntrin::_InterlockedAnd:
555 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E);
556 case MSVCIntrin::_InterlockedExchange:
557 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E);
558 case MSVCIntrin::_InterlockedExchangeAdd:
559 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E);
560 case MSVCIntrin::_InterlockedExchangeSub:
561 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E);
562 case MSVCIntrin::_InterlockedOr:
563 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E);
564 case MSVCIntrin::_InterlockedXor:
565 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E);
566
567 case MSVCIntrin::_interlockedbittestandset: {
568 llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
569 llvm::Value *Bit = EmitScalarExpr(E->getArg(1));
570 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
571 AtomicRMWInst::Or, Addr,
572 Builder.CreateShl(ConstantInt::get(Bit->getType(), 1), Bit),
573 llvm::AtomicOrdering::SequentiallyConsistent);
574 // Shift the relevant bit to the least significant position, truncate to
575 // the result type, and test the low bit.
576 llvm::Value *Shifted = Builder.CreateLShr(RMWI, Bit);
577 llvm::Value *Truncated =
578 Builder.CreateTrunc(Shifted, ConvertType(E->getType()));
579 return Builder.CreateAnd(Truncated,
580 ConstantInt::get(Truncated->getType(), 1));
581 }
582
583 case MSVCIntrin::_InterlockedDecrement: {
584 llvm::Type *IntTy = ConvertType(E->getType());
585 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
586 AtomicRMWInst::Sub,
587 EmitScalarExpr(E->getArg(0)),
588 ConstantInt::get(IntTy, 1),
589 llvm::AtomicOrdering::SequentiallyConsistent);
590 return Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1));
591 }
592 case MSVCIntrin::_InterlockedIncrement: {
593 llvm::Type *IntTy = ConvertType(E->getType());
594 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
595 AtomicRMWInst::Add,
596 EmitScalarExpr(E->getArg(0)),
597 ConstantInt::get(IntTy, 1),
598 llvm::AtomicOrdering::SequentiallyConsistent);
599 return Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1));
600 }
601
602 case MSVCIntrin::__fastfail: {
603 // Request immediate process termination from the kernel. The instruction
604 // sequences to do this are documented on MSDN:
605 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
606 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
607 StringRef Asm, Constraints;
608 switch (ISA) {
609 default:
610 ErrorUnsupported(E, "__fastfail call for this architecture");
611 break;
612 case llvm::Triple::x86:
613 case llvm::Triple::x86_64:
614 Asm = "int $$0x29";
615 Constraints = "{cx}";
616 break;
617 case llvm::Triple::thumb:
618 Asm = "udf #251";
619 Constraints = "{r0}";
620 break;
621 }
622 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty}, false);
623 llvm::InlineAsm *IA =
624 llvm::InlineAsm::get(FTy, Asm, Constraints, /*SideEffects=*/true);
625 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
626 getLLVMContext(), llvm::AttributeList::FunctionIndex,
627 llvm::Attribute::NoReturn);
628 CallSite CS = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
629 CS.setAttributes(NoReturnAttr);
630 return CS.getInstruction();
631 }
632 }
633 llvm_unreachable("Incorrect MSVC intrinsic!")::llvm::llvm_unreachable_internal("Incorrect MSVC intrinsic!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 633)
;
634}
635
636namespace {
637// ARC cleanup for __builtin_os_log_format
638struct CallObjCArcUse final : EHScopeStack::Cleanup {
639 CallObjCArcUse(llvm::Value *object) : object(object) {}
640 llvm::Value *object;
641
642 void Emit(CodeGenFunction &CGF, Flags flags) override {
643 CGF.EmitARCIntrinsicUse(object);
644 }
645};
646}
647
648Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
649 BuiltinCheckKind Kind) {
650 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero)(static_cast <bool> ((Kind == BCK_CLZPassedZero || Kind
== BCK_CTZPassedZero) && "Unsupported builtin check kind"
) ? void (0) : __assert_fail ("(Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero) && \"Unsupported builtin check kind\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 651, __extension__ __PRETTY_FUNCTION__))
651 && "Unsupported builtin check kind")(static_cast <bool> ((Kind == BCK_CLZPassedZero || Kind
== BCK_CTZPassedZero) && "Unsupported builtin check kind"
) ? void (0) : __assert_fail ("(Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero) && \"Unsupported builtin check kind\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 651, __extension__ __PRETTY_FUNCTION__))
;
652
653 Value *ArgValue = EmitScalarExpr(E);
654 if (!SanOpts.has(SanitizerKind::Builtin) || !getTarget().isCLZForZeroUndef())
655 return ArgValue;
656
657 SanitizerScope SanScope(this);
658 Value *Cond = Builder.CreateICmpNE(
659 ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
660 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
661 SanitizerHandler::InvalidBuiltin,
662 {EmitCheckSourceLocation(E->getExprLoc()),
663 llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
664 None);
665 return ArgValue;
666}
667
668/// Get the argument type for arguments to os_log_helper.
669static CanQualType getOSLogArgType(ASTContext &C, int Size) {
670 QualType UnsignedTy = C.getIntTypeForBitwidth(Size * 8, /*Signed=*/false);
671 return C.getCanonicalType(UnsignedTy);
672}
673
674llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
675 const analyze_os_log::OSLogBufferLayout &Layout,
676 CharUnits BufferAlignment) {
677 ASTContext &Ctx = getContext();
678
679 llvm::SmallString<64> Name;
680 {
681 raw_svector_ostream OS(Name);
682 OS << "__os_log_helper";
683 OS << "_" << BufferAlignment.getQuantity();
684 OS << "_" << int(Layout.getSummaryByte());
685 OS << "_" << int(Layout.getNumArgsByte());
686 for (const auto &Item : Layout.Items)
687 OS << "_" << int(Item.getSizeByte()) << "_"
688 << int(Item.getDescriptorByte());
689 }
690
691 if (llvm::Function *F = CGM.getModule().getFunction(Name))
692 return F;
693
694 llvm::SmallVector<ImplicitParamDecl, 4> Params;
695 Params.emplace_back(Ctx, nullptr, SourceLocation(), &Ctx.Idents.get("buffer"),
696 Ctx.VoidPtrTy, ImplicitParamDecl::Other);
697
698 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
699 char Size = Layout.Items[I].getSizeByte();
700 if (!Size)
701 continue;
702
703 Params.emplace_back(
704 Ctx, nullptr, SourceLocation(),
705 &Ctx.Idents.get(std::string("arg") + llvm::to_string(I)),
706 getOSLogArgType(Ctx, Size), ImplicitParamDecl::Other);
707 }
708
709 FunctionArgList Args;
710 for (auto &P : Params)
711 Args.push_back(&P);
712
713 // The helper function has linkonce_odr linkage to enable the linker to merge
714 // identical functions. To ensure the merging always happens, 'noinline' is
715 // attached to the function when compiling with -Oz.
716 const CGFunctionInfo &FI =
717 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
718 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
719 llvm::Function *Fn = llvm::Function::Create(
720 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
721 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
722 CGM.SetLLVMFunctionAttributes(nullptr, FI, Fn);
723 CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Fn);
724
725 // Attach 'noinline' at -Oz.
726 if (CGM.getCodeGenOpts().OptimizeSize == 2)
727 Fn->addFnAttr(llvm::Attribute::NoInline);
728
729 auto NL = ApplyDebugLocation::CreateEmpty(*this);
730 IdentifierInfo *II = &Ctx.Idents.get(Name);
731 FunctionDecl *FD = FunctionDecl::Create(
732 Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(), II,
733 Ctx.VoidTy, nullptr, SC_PrivateExtern, false, false);
734
735 StartFunction(FD, Ctx.VoidTy, Fn, FI, Args);
736
737 // Create a scope with an artificial location for the body of this function.
738 auto AL = ApplyDebugLocation::CreateArtificial(*this);
739
740 CharUnits Offset;
741 Address BufAddr(Builder.CreateLoad(GetAddrOfLocalVar(&Params[0]), "buf"),
742 BufferAlignment);
743 Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
744 Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
745 Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
746 Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
747
748 unsigned I = 1;
749 for (const auto &Item : Layout.Items) {
750 Builder.CreateStore(
751 Builder.getInt8(Item.getDescriptorByte()),
752 Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
753 Builder.CreateStore(
754 Builder.getInt8(Item.getSizeByte()),
755 Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
756
757 CharUnits Size = Item.size();
758 if (!Size.getQuantity())
759 continue;
760
761 Address Arg = GetAddrOfLocalVar(&Params[I]);
762 Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
763 Addr = Builder.CreateBitCast(Addr, Arg.getPointer()->getType(),
764 "argDataCast");
765 Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
766 Offset += Size;
767 ++I;
768 }
769
770 FinishFunction();
771
772 return Fn;
773}
774
775RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
776 assert(E.getNumArgs() >= 2 &&(static_cast <bool> (E.getNumArgs() >= 2 && "__builtin_os_log_format takes at least 2 arguments"
) ? void (0) : __assert_fail ("E.getNumArgs() >= 2 && \"__builtin_os_log_format takes at least 2 arguments\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 777, __extension__ __PRETTY_FUNCTION__))
777 "__builtin_os_log_format takes at least 2 arguments")(static_cast <bool> (E.getNumArgs() >= 2 && "__builtin_os_log_format takes at least 2 arguments"
) ? void (0) : __assert_fail ("E.getNumArgs() >= 2 && \"__builtin_os_log_format takes at least 2 arguments\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 777, __extension__ __PRETTY_FUNCTION__))
;
778 ASTContext &Ctx = getContext();
779 analyze_os_log::OSLogBufferLayout Layout;
780 analyze_os_log::computeOSLogBufferLayout(Ctx, &E, Layout);
781 Address BufAddr = EmitPointerWithAlignment(E.getArg(0));
782 llvm::SmallVector<llvm::Value *, 4> RetainableOperands;
783
784 // Ignore argument 1, the format string. It is not currently used.
785 CallArgList Args;
786 Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy);
787
788 for (const auto &Item : Layout.Items) {
789 int Size = Item.getSizeByte();
790 if (!Size)
791 continue;
792
793 llvm::Value *ArgVal;
794
795 if (const Expr *TheExpr = Item.getExpr()) {
796 ArgVal = EmitScalarExpr(TheExpr, /*Ignore*/ false);
797
798 // Check if this is a retainable type.
799 if (TheExpr->getType()->isObjCRetainableType()) {
800 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&(static_cast <bool> (getEvaluationKind(TheExpr->getType
()) == TEK_Scalar && "Only scalar can be a ObjC retainable type"
) ? void (0) : __assert_fail ("getEvaluationKind(TheExpr->getType()) == TEK_Scalar && \"Only scalar can be a ObjC retainable type\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 801, __extension__ __PRETTY_FUNCTION__))
801 "Only scalar can be a ObjC retainable type")(static_cast <bool> (getEvaluationKind(TheExpr->getType
()) == TEK_Scalar && "Only scalar can be a ObjC retainable type"
) ? void (0) : __assert_fail ("getEvaluationKind(TheExpr->getType()) == TEK_Scalar && \"Only scalar can be a ObjC retainable type\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 801, __extension__ __PRETTY_FUNCTION__))
;
802 // Check if the object is constant, if not, save it in
803 // RetainableOperands.
804 if (!isa<Constant>(ArgVal))
805 RetainableOperands.push_back(ArgVal);
806 }
807 } else {
808 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
809 }
810
811 unsigned ArgValSize =
812 CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
813 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
814 ArgValSize);
815 ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
816 CanQualType ArgTy = getOSLogArgType(Ctx, Size);
817 // If ArgVal has type x86_fp80, zero-extend ArgVal.
818 ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
819 Args.add(RValue::get(ArgVal), ArgTy);
820 }
821
822 const CGFunctionInfo &FI =
823 CGM.getTypes().arrangeBuiltinFunctionCall(Ctx.VoidTy, Args);
824 llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
825 Layout, BufAddr.getAlignment());
826 EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args);
827
828 // Push a clang.arc.use cleanup for each object in RetainableOperands. The
829 // cleanup will cause the use to appear after the final log call, keeping
830 // the object valid while it’s held in the log buffer. Note that if there’s
831 // a release cleanup on the object, it will already be active; since
832 // cleanups are emitted in reverse order, the use will occur before the
833 // object is released.
834 if (!RetainableOperands.empty() && getLangOpts().ObjCAutoRefCount &&
835 CGM.getCodeGenOpts().OptimizationLevel != 0)
836 for (llvm::Value *Object : RetainableOperands)
837 pushFullExprCleanup<CallObjCArcUse>(getARCCleanupKind(), Object);
838
839 return RValue::get(BufAddr.getPointer());
840}
841
842RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
843 unsigned BuiltinID, const CallExpr *E,
844 ReturnValueSlot ReturnValue) {
845 // See if we can constant fold this builtin. If so, don't emit it at all.
846 Expr::EvalResult Result;
847 if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
848 !Result.hasSideEffects()) {
849 if (Result.Val.isInt())
850 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
851 Result.Val.getInt()));
852 if (Result.Val.isFloat())
853 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
854 Result.Val.getFloat()));
855 }
856
857 switch (BuiltinID) {
858 default: break; // Handle intrinsics and libm functions below.
859 case Builtin::BI__builtin___CFStringMakeConstantString:
860 case Builtin::BI__builtin___NSStringMakeConstantString:
861 return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
862 case Builtin::BI__builtin_stdarg_start:
863 case Builtin::BI__builtin_va_start:
864 case Builtin::BI__va_start:
865 case Builtin::BI__builtin_va_end:
866 return RValue::get(
867 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
868 ? EmitScalarExpr(E->getArg(0))
869 : EmitVAListRef(E->getArg(0)).getPointer(),
870 BuiltinID != Builtin::BI__builtin_va_end));
871 case Builtin::BI__builtin_va_copy: {
872 Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
873 Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
874
875 llvm::Type *Type = Int8PtrTy;
876
877 DstPtr = Builder.CreateBitCast(DstPtr, Type);
878 SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
879 return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy),
880 {DstPtr, SrcPtr}));
881 }
882 case Builtin::BI__builtin_abs:
883 case Builtin::BI__builtin_labs:
884 case Builtin::BI__builtin_llabs: {
885 Value *ArgValue = EmitScalarExpr(E->getArg(0));
886
887 Value *NegOp = Builder.CreateNeg(ArgValue, "neg");
888 Value *CmpResult =
889 Builder.CreateICmpSGE(ArgValue,
890 llvm::Constant::getNullValue(ArgValue->getType()),
891 "abscond");
892 Value *Result =
893 Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs");
894
895 return RValue::get(Result);
896 }
897 case Builtin::BI__builtin_fabs:
898 case Builtin::BI__builtin_fabsf:
899 case Builtin::BI__builtin_fabsl: {
900 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs));
901 }
902 case Builtin::BI__builtin_fmod:
903 case Builtin::BI__builtin_fmodf:
904 case Builtin::BI__builtin_fmodl: {
905 Value *Arg1 = EmitScalarExpr(E->getArg(0));
906 Value *Arg2 = EmitScalarExpr(E->getArg(1));
907 Value *Result = Builder.CreateFRem(Arg1, Arg2, "fmod");
908 return RValue::get(Result);
909 }
910 case Builtin::BI__builtin_copysign:
911 case Builtin::BI__builtin_copysignf:
912 case Builtin::BI__builtin_copysignl: {
913 return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::copysign));
914 }
915 case Builtin::BI__builtin_ceil:
916 case Builtin::BI__builtin_ceilf:
917 case Builtin::BI__builtin_ceill: {
918 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::ceil));
919 }
920 case Builtin::BI__builtin_floor:
921 case Builtin::BI__builtin_floorf:
922 case Builtin::BI__builtin_floorl: {
923 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::floor));
924 }
925 case Builtin::BI__builtin_trunc:
926 case Builtin::BI__builtin_truncf:
927 case Builtin::BI__builtin_truncl: {
928 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::trunc));
929 }
930 case Builtin::BI__builtin_rint:
931 case Builtin::BI__builtin_rintf:
932 case Builtin::BI__builtin_rintl: {
933 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::rint));
934 }
935 case Builtin::BI__builtin_nearbyint:
936 case Builtin::BI__builtin_nearbyintf:
937 case Builtin::BI__builtin_nearbyintl: {
938 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::nearbyint));
939 }
940 case Builtin::BI__builtin_round:
941 case Builtin::BI__builtin_roundf:
942 case Builtin::BI__builtin_roundl: {
943 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::round));
944 }
945 case Builtin::BI__builtin_fmin:
946 case Builtin::BI__builtin_fminf:
947 case Builtin::BI__builtin_fminl: {
948 return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::minnum));
949 }
950 case Builtin::BI__builtin_fmax:
951 case Builtin::BI__builtin_fmaxf:
952 case Builtin::BI__builtin_fmaxl: {
953 return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::maxnum));
954 }
955 case Builtin::BI__builtin_conj:
956 case Builtin::BI__builtin_conjf:
957 case Builtin::BI__builtin_conjl: {
958 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
959 Value *Real = ComplexVal.first;
960 Value *Imag = ComplexVal.second;
961 Value *Zero =
962 Imag->getType()->isFPOrFPVectorTy()
963 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
964 : llvm::Constant::getNullValue(Imag->getType());
965
966 Imag = Builder.CreateFSub(Zero, Imag, "sub");
967 return RValue::getComplex(std::make_pair(Real, Imag));
968 }
969 case Builtin::BI__builtin_creal:
970 case Builtin::BI__builtin_crealf:
971 case Builtin::BI__builtin_creall:
972 case Builtin::BIcreal:
973 case Builtin::BIcrealf:
974 case Builtin::BIcreall: {
975 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
976 return RValue::get(ComplexVal.first);
977 }
978
979 case Builtin::BI__builtin_cimag:
980 case Builtin::BI__builtin_cimagf:
981 case Builtin::BI__builtin_cimagl:
982 case Builtin::BIcimag:
983 case Builtin::BIcimagf:
984 case Builtin::BIcimagl: {
985 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
986 return RValue::get(ComplexVal.second);
987 }
988
989 case Builtin::BI__builtin_ctzs:
990 case Builtin::BI__builtin_ctz:
991 case Builtin::BI__builtin_ctzl:
992 case Builtin::BI__builtin_ctzll: {
993 Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero);
994
995 llvm::Type *ArgType = ArgValue->getType();
996 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
997
998 llvm::Type *ResultType = ConvertType(E->getType());
999 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1000 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1001 if (Result->getType() != ResultType)
1002 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1003 "cast");
1004 return RValue::get(Result);
1005 }
1006 case Builtin::BI__builtin_clzs:
1007 case Builtin::BI__builtin_clz:
1008 case Builtin::BI__builtin_clzl:
1009 case Builtin::BI__builtin_clzll: {
1010 Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero);
1011
1012 llvm::Type *ArgType = ArgValue->getType();
1013 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1014
1015 llvm::Type *ResultType = ConvertType(E->getType());
1016 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1017 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1018 if (Result->getType() != ResultType)
1019 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1020 "cast");
1021 return RValue::get(Result);
1022 }
1023 case Builtin::BI__builtin_ffs:
1024 case Builtin::BI__builtin_ffsl:
1025 case Builtin::BI__builtin_ffsll: {
1026 // ffs(x) -> x ? cttz(x) + 1 : 0
1027 Value *ArgValue = EmitScalarExpr(E->getArg(0));
1028
1029 llvm::Type *ArgType = ArgValue->getType();
1030 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1031
1032 llvm::Type *ResultType = ConvertType(E->getType());
1033 Value *Tmp =
1034 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
1035 llvm::ConstantInt::get(ArgType, 1));
1036 Value *Zero = llvm::Constant::getNullValue(ArgType);
1037 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
1038 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
1039 if (Result->getType() != ResultType)
1040 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1041 "cast");
1042 return RValue::get(Result);
1043 }
1044 case Builtin::BI__builtin_parity:
1045 case Builtin::BI__builtin_parityl:
1046 case Builtin::BI__builtin_parityll: {
1047 // parity(x) -> ctpop(x) & 1
1048 Value *ArgValue = EmitScalarExpr(E->getArg(0));
1049
1050 llvm::Type *ArgType = ArgValue->getType();
1051 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1052
1053 llvm::Type *ResultType = ConvertType(E->getType());
1054 Value *Tmp = Builder.CreateCall(F, ArgValue);
1055 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
1056 if (Result->getType() != ResultType)
1057 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1058 "cast");
1059 return RValue::get(Result);
1060 }
1061 case Builtin::BI__popcnt16:
1062 case Builtin::BI__popcnt:
1063 case Builtin::BI__popcnt64:
1064 case Builtin::BI__builtin_popcount:
1065 case Builtin::BI__builtin_popcountl:
1066 case Builtin::BI__builtin_popcountll: {
1067 Value *ArgValue = EmitScalarExpr(E->getArg(0));
1068
1069 llvm::Type *ArgType = ArgValue->getType();
1070 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1071
1072 llvm::Type *ResultType = ConvertType(E->getType());
1073 Value *Result = Builder.CreateCall(F, ArgValue);
1074 if (Result->getType() != ResultType)
1075 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1076 "cast");
1077 return RValue::get(Result);
1078 }
1079 case Builtin::BI_rotr8:
1080 case Builtin::BI_rotr16:
1081 case Builtin::BI_rotr:
1082 case Builtin::BI_lrotr:
1083 case Builtin::BI_rotr64: {
1084 Value *Val = EmitScalarExpr(E->getArg(0));
1085 Value *Shift = EmitScalarExpr(E->getArg(1));
1086
1087 llvm::Type *ArgType = Val->getType();
1088 Shift = Builder.CreateIntCast(Shift, ArgType, false);
1089 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1090 Value *ArgTypeSize = llvm::ConstantInt::get(ArgType, ArgWidth);
1091 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1092
1093 Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
1094 Shift = Builder.CreateAnd(Shift, Mask);
1095 Value *LeftShift = Builder.CreateSub(ArgTypeSize, Shift);
1096
1097 Value *RightShifted = Builder.CreateLShr(Val, Shift);
1098 Value *LeftShifted = Builder.CreateShl(Val, LeftShift);
1099 Value *Rotated = Builder.CreateOr(LeftShifted, RightShifted);
1100
1101 Value *ShiftIsZero = Builder.CreateICmpEQ(Shift, ArgZero);
1102 Value *Result = Builder.CreateSelect(ShiftIsZero, Val, Rotated);
1103 return RValue::get(Result);
1104 }
1105 case Builtin::BI_rotl8:
1106 case Builtin::BI_rotl16:
1107 case Builtin::BI_rotl:
1108 case Builtin::BI_lrotl:
1109 case Builtin::BI_rotl64: {
1110 Value *Val = EmitScalarExpr(E->getArg(0));
1111 Value *Shift = EmitScalarExpr(E->getArg(1));
1112
1113 llvm::Type *ArgType = Val->getType();
1114 Shift = Builder.CreateIntCast(Shift, ArgType, false);
1115 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1116 Value *ArgTypeSize = llvm::ConstantInt::get(ArgType, ArgWidth);
1117 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1118
1119 Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
1120 Shift = Builder.CreateAnd(Shift, Mask);
1121 Value *RightShift = Builder.CreateSub(ArgTypeSize, Shift);
1122
1123 Value *LeftShifted = Builder.CreateShl(Val, Shift);
1124 Value *RightShifted = Builder.CreateLShr(Val, RightShift);
1125 Value *Rotated = Builder.CreateOr(LeftShifted, RightShifted);
1126
1127 Value *ShiftIsZero = Builder.CreateICmpEQ(Shift, ArgZero);
1128 Value *Result = Builder.CreateSelect(ShiftIsZero, Val, Rotated);
1129 return RValue::get(Result);
1130 }
1131 case Builtin::BI__builtin_unpredictable: {
1132 // Always return the argument of __builtin_unpredictable. LLVM does not
1133 // handle this builtin. Metadata for this builtin should be added directly
1134 // to instructions such as branches or switches that use it.
1135 return RValue::get(EmitScalarExpr(E->getArg(0)));
1136 }
1137 case Builtin::BI__builtin_expect: {
1138 Value *ArgValue = EmitScalarExpr(E->getArg(0));
1139 llvm::Type *ArgType = ArgValue->getType();
1140
1141 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
1142 // Don't generate llvm.expect on -O0 as the backend won't use it for
1143 // anything.
1144 // Note, we still IRGen ExpectedValue because it could have side-effects.
1145 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1146 return RValue::get(ArgValue);
1147
1148 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
1149 Value *Result =
1150 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
1151 return RValue::get(Result);
1152 }
1153 case Builtin::BI__builtin_assume_aligned: {
1154 Value *PtrValue = EmitScalarExpr(E->getArg(0));
1155 Value *OffsetValue =
1156 (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
1157
1158 Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
1159 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
1160 unsigned Alignment = (unsigned) AlignmentCI->getZExtValue();
1161
1162 EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue);
1163 return RValue::get(PtrValue);
1164 }
1165 case Builtin::BI__assume:
1166 case Builtin::BI__builtin_assume: {
1167 if (E->getArg(0)->HasSideEffects(getContext()))
1168 return RValue::get(nullptr);
1169
1170 Value *ArgValue = EmitScalarExpr(E->getArg(0));
1171 Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
1172 return RValue::get(Builder.CreateCall(FnAssume, ArgValue));
1173 }
1174 case Builtin::BI__builtin_bswap16:
1175 case Builtin::BI__builtin_bswap32:
1176 case Builtin::BI__builtin_bswap64: {
1177 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap));
1178 }
1179 case Builtin::BI__builtin_bitreverse8:
1180 case Builtin::BI__builtin_bitreverse16:
1181 case Builtin::BI__builtin_bitreverse32:
1182 case Builtin::BI__builtin_bitreverse64: {
1183 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse));
1184 }
1185 case Builtin::BI__builtin_object_size: {
1186 unsigned Type =
1187 E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
1188 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
1189
1190 // We pass this builtin onto the optimizer so that it can figure out the
1191 // object size in more complex cases.
1192 return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType,
1193 /*EmittedE=*/nullptr));
1194 }
1195 case Builtin::BI__builtin_prefetch: {
1196 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
1197 // FIXME: Technically these constants should of type 'int', yes?
1198 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
1199 llvm::ConstantInt::get(Int32Ty, 0);
1200 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
1201 llvm::ConstantInt::get(Int32Ty, 3);
1202 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
1203 Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
1204 return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
1205 }
1206 case Builtin::BI__builtin_readcyclecounter: {
1207 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
1208 return RValue::get(Builder.CreateCall(F));
1209 }
1210 case Builtin::BI__builtin___clear_cache: {
1211 Value *Begin = EmitScalarExpr(E->getArg(0));
1212 Value *End = EmitScalarExpr(E->getArg(1));
1213 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
1214 return RValue::get(Builder.CreateCall(F, {Begin, End}));
1215 }
1216 case Builtin::BI__builtin_trap:
1217 return RValue::get(EmitTrapCall(Intrinsic::trap));
1218 case Builtin::BI__debugbreak:
1219 return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
1220 case Builtin::BI__builtin_unreachable: {
1221 if (SanOpts.has(SanitizerKind::Unreachable)) {
1222 SanitizerScope SanScope(this);
1223 EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
1224 SanitizerKind::Unreachable),
1225 SanitizerHandler::BuiltinUnreachable,
1226 EmitCheckSourceLocation(E->getExprLoc()), None);
1227 } else
1228 Builder.CreateUnreachable();
1229
1230 // We do need to preserve an insertion point.
1231 EmitBlock(createBasicBlock("unreachable.cont"));
1232
1233 return RValue::get(nullptr);
1234 }
1235
1236 case Builtin::BI__builtin_powi:
1237 case Builtin::BI__builtin_powif:
1238 case Builtin::BI__builtin_powil: {
1239 Value *Base = EmitScalarExpr(E->getArg(0));
1240 Value *Exponent = EmitScalarExpr(E->getArg(1));
1241 llvm::Type *ArgType = Base->getType();
1242 Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
1243 return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
1244 }
1245
1246 case Builtin::BI__builtin_isgreater:
1247 case Builtin::BI__builtin_isgreaterequal:
1248 case Builtin::BI__builtin_isless:
1249 case Builtin::BI__builtin_islessequal:
1250 case Builtin::BI__builtin_islessgreater:
1251 case Builtin::BI__builtin_isunordered: {
1252 // Ordered comparisons: we know the arguments to these are matching scalar
1253 // floating point values.
1254 Value *LHS = EmitScalarExpr(E->getArg(0));
1255 Value *RHS = EmitScalarExpr(E->getArg(1));
1256
1257 switch (BuiltinID) {
1258 default: llvm_unreachable("Unknown ordered comparison")::llvm::llvm_unreachable_internal("Unknown ordered comparison"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 1258)
;
1259 case Builtin::BI__builtin_isgreater:
1260 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
1261 break;
1262 case Builtin::BI__builtin_isgreaterequal:
1263 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
1264 break;
1265 case Builtin::BI__builtin_isless:
1266 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
1267 break;
1268 case Builtin::BI__builtin_islessequal:
1269 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
1270 break;
1271 case Builtin::BI__builtin_islessgreater:
1272 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
1273 break;
1274 case Builtin::BI__builtin_isunordered:
1275 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
1276 break;
1277 }
1278 // ZExt bool to int type.
1279 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
1280 }
1281 case Builtin::BI__builtin_isnan: {
1282 Value *V = EmitScalarExpr(E->getArg(0));
1283 V = Builder.CreateFCmpUNO(V, V, "cmp");
1284 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
1285 }
1286
1287 case Builtin::BIfinite:
1288 case Builtin::BI__finite:
1289 case Builtin::BIfinitef:
1290 case Builtin::BI__finitef:
1291 case Builtin::BIfinitel:
1292 case Builtin::BI__finitel:
1293 case Builtin::BI__builtin_isinf:
1294 case Builtin::BI__builtin_isfinite: {
1295 // isinf(x) --> fabs(x) == infinity
1296 // isfinite(x) --> fabs(x) != infinity
1297 // x != NaN via the ordered compare in either case.
1298 Value *V = EmitScalarExpr(E->getArg(0));
1299 Value *Fabs = EmitFAbs(*this, V);
1300 Constant *Infinity = ConstantFP::getInfinity(V->getType());
1301 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
1302 ? CmpInst::FCMP_OEQ
1303 : CmpInst::FCMP_ONE;
1304 Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf");
1305 return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType())));
1306 }
1307
1308 case Builtin::BI__builtin_isinf_sign: {
1309 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
1310 Value *Arg = EmitScalarExpr(E->getArg(0));
1311 Value *AbsArg = EmitFAbs(*this, Arg);
1312 Value *IsInf = Builder.CreateFCmpOEQ(
1313 AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
1314 Value *IsNeg = EmitSignBit(*this, Arg);
1315
1316 llvm::Type *IntTy = ConvertType(E->getType());
1317 Value *Zero = Constant::getNullValue(IntTy);
1318 Value *One = ConstantInt::get(IntTy, 1);
1319 Value *NegativeOne = ConstantInt::get(IntTy, -1);
1320 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
1321 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
1322 return RValue::get(Result);
1323 }
1324
1325 case Builtin::BI__builtin_isnormal: {
1326 // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
1327 Value *V = EmitScalarExpr(E->getArg(0));
1328 Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
1329
1330 Value *Abs = EmitFAbs(*this, V);
1331 Value *IsLessThanInf =
1332 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
1333 APFloat Smallest = APFloat::getSmallestNormalized(
1334 getContext().getFloatTypeSemantics(E->getArg(0)->getType()));
1335 Value *IsNormal =
1336 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
1337 "isnormal");
1338 V = Builder.CreateAnd(Eq, IsLessThanInf, "and");
1339 V = Builder.CreateAnd(V, IsNormal, "and");
1340 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
1341 }
1342
1343 case Builtin::BI__builtin_fpclassify: {
1344 Value *V = EmitScalarExpr(E->getArg(5));
1345 llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
1346
1347 // Create Result
1348 BasicBlock *Begin = Builder.GetInsertBlock();
1349 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
1350 Builder.SetInsertPoint(End);
1351 PHINode *Result =
1352 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
1353 "fpclassify_result");
1354
1355 // if (V==0) return FP_ZERO
1356 Builder.SetInsertPoint(Begin);
1357 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
1358 "iszero");
1359 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
1360 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
1361 Builder.CreateCondBr(IsZero, End, NotZero);
1362 Result->addIncoming(ZeroLiteral, Begin);
1363
1364 // if (V != V) return FP_NAN
1365 Builder.SetInsertPoint(NotZero);
1366 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
1367 Value *NanLiteral = EmitScalarExpr(E->getArg(0));
1368 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
1369 Builder.CreateCondBr(IsNan, End, NotNan);
1370 Result->addIncoming(NanLiteral, NotZero);
1371
1372 // if (fabs(V) == infinity) return FP_INFINITY
1373 Builder.SetInsertPoint(NotNan);
1374 Value *VAbs = EmitFAbs(*this, V);
1375 Value *IsInf =
1376 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
1377 "isinf");
1378 Value *InfLiteral = EmitScalarExpr(E->getArg(1));
1379 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
1380 Builder.CreateCondBr(IsInf, End, NotInf);
1381 Result->addIncoming(InfLiteral, NotNan);
1382
1383 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
1384 Builder.SetInsertPoint(NotInf);
1385 APFloat Smallest = APFloat::getSmallestNormalized(
1386 getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
1387 Value *IsNormal =
1388 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
1389 "isnormal");
1390 Value *NormalResult =
1391 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
1392 EmitScalarExpr(E->getArg(3)));
1393 Builder.CreateBr(End);
1394 Result->addIncoming(NormalResult, NotInf);
1395
1396 // return Result
1397 Builder.SetInsertPoint(End);
1398 return RValue::get(Result);
1399 }
1400
1401 case Builtin::BIalloca:
1402 case Builtin::BI_alloca:
1403 case Builtin::BI__builtin_alloca: {
1404 Value *Size = EmitScalarExpr(E->getArg(0));
1405 const TargetInfo &TI = getContext().getTargetInfo();
1406 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
1407 unsigned SuitableAlignmentInBytes =
1408 CGM.getContext()
1409 .toCharUnitsFromBits(TI.getSuitableAlign())
1410 .getQuantity();
1411 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1412 AI->setAlignment(SuitableAlignmentInBytes);
1413 return RValue::get(AI);
1414 }
1415
1416 case Builtin::BI__builtin_alloca_with_align: {
1417 Value *Size = EmitScalarExpr(E->getArg(0));
1418 Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
1419 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
1420 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
1421 unsigned AlignmentInBytes =
1422 CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getQuantity();
1423 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1424 AI->setAlignment(AlignmentInBytes);
1425 return RValue::get(AI);
1426 }
1427
1428 case Builtin::BIbzero:
1429 case Builtin::BI__builtin_bzero: {
1430 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1431 Value *SizeVal = EmitScalarExpr(E->getArg(1));
1432 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1433 E->getArg(0)->getExprLoc(), FD, 0);
1434 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
1435 return RValue::get(nullptr);
1436 }
1437 case Builtin::BImemcpy:
1438 case Builtin::BI__builtin_memcpy: {
1439 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1440 Address Src = EmitPointerWithAlignment(E->getArg(1));
1441 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1442 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1443 E->getArg(0)->getExprLoc(), FD, 0);
1444 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
1445 E->getArg(1)->getExprLoc(), FD, 1);
1446 Builder.CreateMemCpy(Dest, Src, SizeVal, false);
1447 return RValue::get(Dest.getPointer());
1448 }
1449
1450 case Builtin::BI__builtin_char_memchr:
1451 BuiltinID = Builtin::BI__builtin_memchr;
1452 break;
1453
1454 case Builtin::BI__builtin___memcpy_chk: {
1455 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
1456 llvm::APSInt Size, DstSize;
1457 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1458 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1459 break;
1460 if (Size.ugt(DstSize))
1461 break;
1462 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1463 Address Src = EmitPointerWithAlignment(E->getArg(1));
1464 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1465 Builder.CreateMemCpy(Dest, Src, SizeVal, false);
1466 return RValue::get(Dest.getPointer());
1467 }
1468
1469 case Builtin::BI__builtin_objc_memmove_collectable: {
1470 Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
1471 Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
1472 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1473 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
1474 DestAddr, SrcAddr, SizeVal);
1475 return RValue::get(DestAddr.getPointer());
1476 }
1477
1478 case Builtin::BI__builtin___memmove_chk: {
1479 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
1480 llvm::APSInt Size, DstSize;
1481 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1482 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1483 break;
1484 if (Size.ugt(DstSize))
1485 break;
1486 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1487 Address Src = EmitPointerWithAlignment(E->getArg(1));
1488 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1489 Builder.CreateMemMove(Dest, Src, SizeVal, false);
1490 return RValue::get(Dest.getPointer());
1491 }
1492
1493 case Builtin::BImemmove:
1494 case Builtin::BI__builtin_memmove: {
1495 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1496 Address Src = EmitPointerWithAlignment(E->getArg(1));
1497 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1498 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1499 E->getArg(0)->getExprLoc(), FD, 0);
1500 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
1501 E->getArg(1)->getExprLoc(), FD, 1);
1502 Builder.CreateMemMove(Dest, Src, SizeVal, false);
1503 return RValue::get(Dest.getPointer());
1504 }
1505 case Builtin::BImemset:
1506 case Builtin::BI__builtin_memset: {
1507 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1508 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1509 Builder.getInt8Ty());
1510 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1511 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1512 E->getArg(0)->getExprLoc(), FD, 0);
1513 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
1514 return RValue::get(Dest.getPointer());
1515 }
1516 case Builtin::BI__builtin___memset_chk: {
1517 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
1518 llvm::APSInt Size, DstSize;
1519 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1520 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1521 break;
1522 if (Size.ugt(DstSize))
1523 break;
1524 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1525 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1526 Builder.getInt8Ty());
1527 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1528 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
1529 return RValue::get(Dest.getPointer());
1530 }
1531 case Builtin::BI__builtin_dwarf_cfa: {
1532 // The offset in bytes from the first argument to the CFA.
1533 //
1534 // Why on earth is this in the frontend? Is there any reason at
1535 // all that the backend can't reasonably determine this while
1536 // lowering llvm.eh.dwarf.cfa()?
1537 //
1538 // TODO: If there's a satisfactory reason, add a target hook for
1539 // this instead of hard-coding 0, which is correct for most targets.
1540 int32_t Offset = 0;
1541
1542 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
1543 return RValue::get(Builder.CreateCall(F,
1544 llvm::ConstantInt::get(Int32Ty, Offset)));
1545 }
1546 case Builtin::BI__builtin_return_address: {
1547 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
1548 getContext().UnsignedIntTy);
1549 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
1550 return RValue::get(Builder.CreateCall(F, Depth));
1551 }
1552 case Builtin::BI_ReturnAddress: {
1553 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
1554 return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
1555 }
1556 case Builtin::BI__builtin_frame_address: {
1557 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
1558 getContext().UnsignedIntTy);
1559 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
1560 return RValue::get(Builder.CreateCall(F, Depth));
1561 }
1562 case Builtin::BI__builtin_extract_return_addr: {
1563 Value *Address = EmitScalarExpr(E->getArg(0));
1564 Value *Result = getTargetHooks().decodeReturnAddress(*this, Address);
1565 return RValue::get(Result);
1566 }
1567 case Builtin::BI__builtin_frob_return_addr: {
1568 Value *Address = EmitScalarExpr(E->getArg(0));
1569 Value *Result = getTargetHooks().encodeReturnAddress(*this, Address);
1570 return RValue::get(Result);
1571 }
1572 case Builtin::BI__builtin_dwarf_sp_column: {
1573 llvm::IntegerType *Ty
1574 = cast<llvm::IntegerType>(ConvertType(E->getType()));
1575 int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
1576 if (Column == -1) {
1577 CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
1578 return RValue::get(llvm::UndefValue::get(Ty));
1579 }
1580 return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
1581 }
1582 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
1583 Value *Address = EmitScalarExpr(E->getArg(0));
1584 if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
1585 CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
1586 return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
1587 }
1588 case Builtin::BI__builtin_eh_return: {
1589 Value *Int = EmitScalarExpr(E->getArg(0));
1590 Value *Ptr = EmitScalarExpr(E->getArg(1));
1591
1592 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
1593 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&(static_cast <bool> ((IntTy->getBitWidth() == 32 || IntTy
->getBitWidth() == 64) && "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"
) ? void (0) : __assert_fail ("(IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && \"LLVM's __builtin_eh_return only supports 32- and 64-bit variants\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 1594, __extension__ __PRETTY_FUNCTION__))
1594 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants")(static_cast <bool> ((IntTy->getBitWidth() == 32 || IntTy
->getBitWidth() == 64) && "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"
) ? void (0) : __assert_fail ("(IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && \"LLVM's __builtin_eh_return only supports 32- and 64-bit variants\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 1594, __extension__ __PRETTY_FUNCTION__))
;
1595 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
1596 ? Intrinsic::eh_return_i32
1597 : Intrinsic::eh_return_i64);
1598 Builder.CreateCall(F, {Int, Ptr});
1599 Builder.CreateUnreachable();
1600
1601 // We do need to preserve an insertion point.
1602 EmitBlock(createBasicBlock("builtin_eh_return.cont"));
1603
1604 return RValue::get(nullptr);
1605 }
1606 case Builtin::BI__builtin_unwind_init: {
1607 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
1608 return RValue::get(Builder.CreateCall(F));
1609 }
1610 case Builtin::BI__builtin_extend_pointer: {
1611 // Extends a pointer to the size of an _Unwind_Word, which is
1612 // uint64_t on all platforms. Generally this gets poked into a
1613 // register and eventually used as an address, so if the
1614 // addressing registers are wider than pointers and the platform
1615 // doesn't implicitly ignore high-order bits when doing
1616 // addressing, we need to make sure we zext / sext based on
1617 // the platform's expectations.
1618 //
1619 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
1620
1621 // Cast the pointer to intptr_t.
1622 Value *Ptr = EmitScalarExpr(E->getArg(0));
1623 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
1624
1625 // If that's 64 bits, we're done.
1626 if (IntPtrTy->getBitWidth() == 64)
1627 return RValue::get(Result);
1628
1629 // Otherwise, ask the codegen data what to do.
1630 if (getTargetHooks().extendPointerWithSExt())
1631 return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
1632 else
1633 return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
1634 }
1635 case Builtin::BI__builtin_setjmp: {
1636 // Buffer is a void**.
1637 Address Buf = EmitPointerWithAlignment(E->getArg(0));
1638
1639 // Store the frame pointer to the setjmp buffer.
1640 Value *FrameAddr =
1641 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
1642 ConstantInt::get(Int32Ty, 0));
1643 Builder.CreateStore(FrameAddr, Buf);
1644
1645 // Store the stack pointer to the setjmp buffer.
1646 Value *StackAddr =
1647 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
1648 Address StackSaveSlot =
1649 Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize());
1650 Builder.CreateStore(StackAddr, StackSaveSlot);
1651
1652 // Call LLVM's EH setjmp, which is lightweight.
1653 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
1654 Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
1655 return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
1656 }
1657 case Builtin::BI__builtin_longjmp: {
1658 Value *Buf = EmitScalarExpr(E->getArg(0));
1659 Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
1660
1661 // Call LLVM's EH longjmp, which is lightweight.
1662 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
1663
1664 // longjmp doesn't return; mark this as unreachable.
1665 Builder.CreateUnreachable();
1666
1667 // We do need to preserve an insertion point.
1668 EmitBlock(createBasicBlock("longjmp.cont"));
1669
1670 return RValue::get(nullptr);
1671 }
1672 case Builtin::BI__sync_fetch_and_add:
1673 case Builtin::BI__sync_fetch_and_sub:
1674 case Builtin::BI__sync_fetch_and_or:
1675 case Builtin::BI__sync_fetch_and_and:
1676 case Builtin::BI__sync_fetch_and_xor:
1677 case Builtin::BI__sync_fetch_and_nand:
1678 case Builtin::BI__sync_add_and_fetch:
1679 case Builtin::BI__sync_sub_and_fetch:
1680 case Builtin::BI__sync_and_and_fetch:
1681 case Builtin::BI__sync_or_and_fetch:
1682 case Builtin::BI__sync_xor_and_fetch:
1683 case Builtin::BI__sync_nand_and_fetch:
1684 case Builtin::BI__sync_val_compare_and_swap:
1685 case Builtin::BI__sync_bool_compare_and_swap:
1686 case Builtin::BI__sync_lock_test_and_set:
1687 case Builtin::BI__sync_lock_release:
1688 case Builtin::BI__sync_swap:
1689 llvm_unreachable("Shouldn't make it through sema")::llvm::llvm_unreachable_internal("Shouldn't make it through sema"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 1689)
;
1690 case Builtin::BI__sync_fetch_and_add_1:
1691 case Builtin::BI__sync_fetch_and_add_2:
1692 case Builtin::BI__sync_fetch_and_add_4:
1693 case Builtin::BI__sync_fetch_and_add_8:
1694 case Builtin::BI__sync_fetch_and_add_16:
1695 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
1696 case Builtin::BI__sync_fetch_and_sub_1:
1697 case Builtin::BI__sync_fetch_and_sub_2:
1698 case Builtin::BI__sync_fetch_and_sub_4:
1699 case Builtin::BI__sync_fetch_and_sub_8:
1700 case Builtin::BI__sync_fetch_and_sub_16:
1701 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
1702 case Builtin::BI__sync_fetch_and_or_1:
1703 case Builtin::BI__sync_fetch_and_or_2:
1704 case Builtin::BI__sync_fetch_and_or_4:
1705 case Builtin::BI__sync_fetch_and_or_8:
1706 case Builtin::BI__sync_fetch_and_or_16:
1707 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
1708 case Builtin::BI__sync_fetch_and_and_1:
1709 case Builtin::BI__sync_fetch_and_and_2:
1710 case Builtin::BI__sync_fetch_and_and_4:
1711 case Builtin::BI__sync_fetch_and_and_8:
1712 case Builtin::BI__sync_fetch_and_and_16:
1713 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
1714 case Builtin::BI__sync_fetch_and_xor_1:
1715 case Builtin::BI__sync_fetch_and_xor_2:
1716 case Builtin::BI__sync_fetch_and_xor_4:
1717 case Builtin::BI__sync_fetch_and_xor_8:
1718 case Builtin::BI__sync_fetch_and_xor_16:
1719 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
1720 case Builtin::BI__sync_fetch_and_nand_1:
1721 case Builtin::BI__sync_fetch_and_nand_2:
1722 case Builtin::BI__sync_fetch_and_nand_4:
1723 case Builtin::BI__sync_fetch_and_nand_8:
1724 case Builtin::BI__sync_fetch_and_nand_16:
1725 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E);
1726
1727 // Clang extensions: not overloaded yet.
1728 case Builtin::BI__sync_fetch_and_min:
1729 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
1730 case Builtin::BI__sync_fetch_and_max:
1731 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
1732 case Builtin::BI__sync_fetch_and_umin:
1733 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
1734 case Builtin::BI__sync_fetch_and_umax:
1735 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
1736
1737 case Builtin::BI__sync_add_and_fetch_1:
1738 case Builtin::BI__sync_add_and_fetch_2:
1739 case Builtin::BI__sync_add_and_fetch_4:
1740 case Builtin::BI__sync_add_and_fetch_8:
1741 case Builtin::BI__sync_add_and_fetch_16:
1742 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
1743 llvm::Instruction::Add);
1744 case Builtin::BI__sync_sub_and_fetch_1:
1745 case Builtin::BI__sync_sub_and_fetch_2:
1746 case Builtin::BI__sync_sub_and_fetch_4:
1747 case Builtin::BI__sync_sub_and_fetch_8:
1748 case Builtin::BI__sync_sub_and_fetch_16:
1749 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
1750 llvm::Instruction::Sub);
1751 case Builtin::BI__sync_and_and_fetch_1:
1752 case Builtin::BI__sync_and_and_fetch_2:
1753 case Builtin::BI__sync_and_and_fetch_4:
1754 case Builtin::BI__sync_and_and_fetch_8:
1755 case Builtin::BI__sync_and_and_fetch_16:
1756 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
1757 llvm::Instruction::And);
1758 case Builtin::BI__sync_or_and_fetch_1:
1759 case Builtin::BI__sync_or_and_fetch_2:
1760 case Builtin::BI__sync_or_and_fetch_4:
1761 case Builtin::BI__sync_or_and_fetch_8:
1762 case Builtin::BI__sync_or_and_fetch_16:
1763 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
1764 llvm::Instruction::Or);
1765 case Builtin::BI__sync_xor_and_fetch_1:
1766 case Builtin::BI__sync_xor_and_fetch_2:
1767 case Builtin::BI__sync_xor_and_fetch_4:
1768 case Builtin::BI__sync_xor_and_fetch_8:
1769 case Builtin::BI__sync_xor_and_fetch_16:
1770 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
1771 llvm::Instruction::Xor);
1772 case Builtin::BI__sync_nand_and_fetch_1:
1773 case Builtin::BI__sync_nand_and_fetch_2:
1774 case Builtin::BI__sync_nand_and_fetch_4:
1775 case Builtin::BI__sync_nand_and_fetch_8:
1776 case Builtin::BI__sync_nand_and_fetch_16:
1777 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E,
1778 llvm::Instruction::And, true);
1779
1780 case Builtin::BI__sync_val_compare_and_swap_1:
1781 case Builtin::BI__sync_val_compare_and_swap_2:
1782 case Builtin::BI__sync_val_compare_and_swap_4:
1783 case Builtin::BI__sync_val_compare_and_swap_8:
1784 case Builtin::BI__sync_val_compare_and_swap_16:
1785 return RValue::get(MakeAtomicCmpXchgValue(*this, E, false));
1786
1787 case Builtin::BI__sync_bool_compare_and_swap_1:
1788 case Builtin::BI__sync_bool_compare_and_swap_2:
1789 case Builtin::BI__sync_bool_compare_and_swap_4:
1790 case Builtin::BI__sync_bool_compare_and_swap_8:
1791 case Builtin::BI__sync_bool_compare_and_swap_16:
1792 return RValue::get(MakeAtomicCmpXchgValue(*this, E, true));
1793
1794 case Builtin::BI__sync_swap_1:
1795 case Builtin::BI__sync_swap_2:
1796 case Builtin::BI__sync_swap_4:
1797 case Builtin::BI__sync_swap_8:
1798 case Builtin::BI__sync_swap_16:
1799 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
1800
1801 case Builtin::BI__sync_lock_test_and_set_1:
1802 case Builtin::BI__sync_lock_test_and_set_2:
1803 case Builtin::BI__sync_lock_test_and_set_4:
1804 case Builtin::BI__sync_lock_test_and_set_8:
1805 case Builtin::BI__sync_lock_test_and_set_16:
1806 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
1807
1808 case Builtin::BI__sync_lock_release_1:
1809 case Builtin::BI__sync_lock_release_2:
1810 case Builtin::BI__sync_lock_release_4:
1811 case Builtin::BI__sync_lock_release_8:
1812 case Builtin::BI__sync_lock_release_16: {
1813 Value *Ptr = EmitScalarExpr(E->getArg(0));
1814 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
1815 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
1816 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
1817 StoreSize.getQuantity() * 8);
1818 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
1819 llvm::StoreInst *Store =
1820 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
1821 StoreSize);
1822 Store->setAtomic(llvm::AtomicOrdering::Release);
1823 return RValue::get(nullptr);
1824 }
1825
1826 case Builtin::BI__sync_synchronize: {
1827 // We assume this is supposed to correspond to a C++0x-style
1828 // sequentially-consistent fence (i.e. this is only usable for
1829 // synchonization, not device I/O or anything like that). This intrinsic
1830 // is really badly designed in the sense that in theory, there isn't
1831 // any way to safely use it... but in practice, it mostly works
1832 // to use it with non-atomic loads and stores to get acquire/release
1833 // semantics.
1834 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
1835 return RValue::get(nullptr);
1836 }
1837
1838 case Builtin::BI__builtin_nontemporal_load:
1839 return RValue::get(EmitNontemporalLoad(*this, E));
1840 case Builtin::BI__builtin_nontemporal_store:
1841 return RValue::get(EmitNontemporalStore(*this, E));
1842 case Builtin::BI__c11_atomic_is_lock_free:
1843 case Builtin::BI__atomic_is_lock_free: {
1844 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
1845 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
1846 // _Atomic(T) is always properly-aligned.
1847 const char *LibCallName = "__atomic_is_lock_free";
1848 CallArgList Args;
1849 Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
1850 getContext().getSizeType());
1851 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
1852 Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
1853 getContext().VoidPtrTy);
1854 else
1855 Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
1856 getContext().VoidPtrTy);
1857 const CGFunctionInfo &FuncInfo =
1858 CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args);
1859 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
1860 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
1861 return EmitCall(FuncInfo, CGCallee::forDirect(Func),
1862 ReturnValueSlot(), Args);
1863 }
1864
1865 case Builtin::BI__atomic_test_and_set: {
1866 // Look at the argument type to determine whether this is a volatile
1867 // operation. The parameter type is always volatile.
1868 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1869 bool Volatile =
1870 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1871
1872 Value *Ptr = EmitScalarExpr(E->getArg(0));
1873 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
1874 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1875 Value *NewVal = Builder.getInt8(1);
1876 Value *Order = EmitScalarExpr(E->getArg(1));
1877 if (isa<llvm::ConstantInt>(Order)) {
1878 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1879 AtomicRMWInst *Result = nullptr;
1880 switch (ord) {
1881 case 0: // memory_order_relaxed
1882 default: // invalid order
1883 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1884 llvm::AtomicOrdering::Monotonic);
1885 break;
1886 case 1: // memory_order_consume
1887 case 2: // memory_order_acquire
1888 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1889 llvm::AtomicOrdering::Acquire);
1890 break;
1891 case 3: // memory_order_release
1892 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1893 llvm::AtomicOrdering::Release);
1894 break;
1895 case 4: // memory_order_acq_rel
1896
1897 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1898 llvm::AtomicOrdering::AcquireRelease);
1899 break;
1900 case 5: // memory_order_seq_cst
1901 Result = Builder.CreateAtomicRMW(
1902 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1903 llvm::AtomicOrdering::SequentiallyConsistent);
1904 break;
1905 }
1906 Result->setVolatile(Volatile);
1907 return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1908 }
1909
1910 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1911
1912 llvm::BasicBlock *BBs[5] = {
1913 createBasicBlock("monotonic", CurFn),
1914 createBasicBlock("acquire", CurFn),
1915 createBasicBlock("release", CurFn),
1916 createBasicBlock("acqrel", CurFn),
1917 createBasicBlock("seqcst", CurFn)
1918 };
1919 llvm::AtomicOrdering Orders[5] = {
1920 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
1921 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
1922 llvm::AtomicOrdering::SequentiallyConsistent};
1923
1924 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1925 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1926
1927 Builder.SetInsertPoint(ContBB);
1928 PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
1929
1930 for (unsigned i = 0; i < 5; ++i) {
1931 Builder.SetInsertPoint(BBs[i]);
1932 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1933 Ptr, NewVal, Orders[i]);
1934 RMW->setVolatile(Volatile);
1935 Result->addIncoming(RMW, BBs[i]);
1936 Builder.CreateBr(ContBB);
1937 }
1938
1939 SI->addCase(Builder.getInt32(0), BBs[0]);
1940 SI->addCase(Builder.getInt32(1), BBs[1]);
1941 SI->addCase(Builder.getInt32(2), BBs[1]);
1942 SI->addCase(Builder.getInt32(3), BBs[2]);
1943 SI->addCase(Builder.getInt32(4), BBs[3]);
1944 SI->addCase(Builder.getInt32(5), BBs[4]);
1945
1946 Builder.SetInsertPoint(ContBB);
1947 return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1948 }
1949
1950 case Builtin::BI__atomic_clear: {
1951 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1952 bool Volatile =
1953 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1954
1955 Address Ptr = EmitPointerWithAlignment(E->getArg(0));
1956 unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace();
1957 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1958 Value *NewVal = Builder.getInt8(0);
1959 Value *Order = EmitScalarExpr(E->getArg(1));
1960 if (isa<llvm::ConstantInt>(Order)) {
1961 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1962 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1963 switch (ord) {
1964 case 0: // memory_order_relaxed
1965 default: // invalid order
1966 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
1967 break;
1968 case 3: // memory_order_release
1969 Store->setOrdering(llvm::AtomicOrdering::Release);
1970 break;
1971 case 5: // memory_order_seq_cst
1972 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
1973 break;
1974 }
1975 return RValue::get(nullptr);
1976 }
1977
1978 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1979
1980 llvm::BasicBlock *BBs[3] = {
1981 createBasicBlock("monotonic", CurFn),
1982 createBasicBlock("release", CurFn),
1983 createBasicBlock("seqcst", CurFn)
1984 };
1985 llvm::AtomicOrdering Orders[3] = {
1986 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
1987 llvm::AtomicOrdering::SequentiallyConsistent};
1988
1989 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1990 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1991
1992 for (unsigned i = 0; i < 3; ++i) {
1993 Builder.SetInsertPoint(BBs[i]);
1994 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1995 Store->setOrdering(Orders[i]);
1996 Builder.CreateBr(ContBB);
1997 }
1998
1999 SI->addCase(Builder.getInt32(0), BBs[0]);
2000 SI->addCase(Builder.getInt32(3), BBs[1]);
2001 SI->addCase(Builder.getInt32(5), BBs[2]);
2002
2003 Builder.SetInsertPoint(ContBB);
2004 return RValue::get(nullptr);
2005 }
2006
2007 case Builtin::BI__atomic_thread_fence:
2008 case Builtin::BI__atomic_signal_fence:
2009 case Builtin::BI__c11_atomic_thread_fence:
2010 case Builtin::BI__c11_atomic_signal_fence: {
2011 llvm::SyncScope::ID SSID;
2012 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
2013 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
2014 SSID = llvm::SyncScope::SingleThread;
2015 else
2016 SSID = llvm::SyncScope::System;
2017 Value *Order = EmitScalarExpr(E->getArg(0));
2018 if (isa<llvm::ConstantInt>(Order)) {
2019 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2020 switch (ord) {
2021 case 0: // memory_order_relaxed
2022 default: // invalid order
2023 break;
2024 case 1: // memory_order_consume
2025 case 2: // memory_order_acquire
2026 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2027 break;
2028 case 3: // memory_order_release
2029 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2030 break;
2031 case 4: // memory_order_acq_rel
2032 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2033 break;
2034 case 5: // memory_order_seq_cst
2035 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2036 break;
2037 }
2038 return RValue::get(nullptr);
2039 }
2040
2041 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
2042 AcquireBB = createBasicBlock("acquire", CurFn);
2043 ReleaseBB = createBasicBlock("release", CurFn);
2044 AcqRelBB = createBasicBlock("acqrel", CurFn);
2045 SeqCstBB = createBasicBlock("seqcst", CurFn);
2046 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
2047
2048 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
2049 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
2050
2051 Builder.SetInsertPoint(AcquireBB);
2052 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2053 Builder.CreateBr(ContBB);
2054 SI->addCase(Builder.getInt32(1), AcquireBB);
2055 SI->addCase(Builder.getInt32(2), AcquireBB);
2056
2057 Builder.SetInsertPoint(ReleaseBB);
2058 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2059 Builder.CreateBr(ContBB);
2060 SI->addCase(Builder.getInt32(3), ReleaseBB);
2061
2062 Builder.SetInsertPoint(AcqRelBB);
2063 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2064 Builder.CreateBr(ContBB);
2065 SI->addCase(Builder.getInt32(4), AcqRelBB);
2066
2067 Builder.SetInsertPoint(SeqCstBB);
2068 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2069 Builder.CreateBr(ContBB);
2070 SI->addCase(Builder.getInt32(5), SeqCstBB);
2071
2072 Builder.SetInsertPoint(ContBB);
2073 return RValue::get(nullptr);
2074 }
2075
2076 case Builtin::BIsqrt:
2077 case Builtin::BIsqrtf:
2078 case Builtin::BIsqrtl:
2079 // Builtins have the same semantics as library functions. The LLVM intrinsic
2080 // has the same semantics as the library function except it does not set
2081 // errno. Thus, we can transform either sqrt or __builtin_sqrt to @llvm.sqrt
2082 // if the call is 'const' (the call must not set errno).
2083 //
2084 // FIXME: The builtin cases are not here because they are marked 'const' in
2085 // Builtins.def. So that means they are wrongly defined to have different
2086 // semantics than the library functions. If we included them here, we would
2087 // turn them into LLVM intrinsics regardless of whether -fmath-errno was on.
2088 if (FD->hasAttr<ConstAttr>())
2089 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::sqrt));
2090 break;
2091
2092 case Builtin::BI__builtin_pow:
2093 case Builtin::BI__builtin_powf:
2094 case Builtin::BI__builtin_powl:
2095 case Builtin::BIpow:
2096 case Builtin::BIpowf:
2097 case Builtin::BIpowl: {
2098 // Transform a call to pow* into a @llvm.pow.* intrinsic call.
2099 if (!FD->hasAttr<ConstAttr>())
2100 break;
2101 Value *Base = EmitScalarExpr(E->getArg(0));
2102 Value *Exponent = EmitScalarExpr(E->getArg(1));
2103 llvm::Type *ArgType = Base->getType();
2104 Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType);
2105 return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
2106 }
2107
2108 case Builtin::BIfma:
2109 case Builtin::BIfmaf:
2110 case Builtin::BIfmal:
2111 case Builtin::BI__builtin_fma:
2112 case Builtin::BI__builtin_fmaf:
2113 case Builtin::BI__builtin_fmal:
2114 // A constant libcall or builtin is equivalent to the LLVM intrinsic.
2115 if (FD->hasAttr<ConstAttr>())
2116 return RValue::get(emitTernaryBuiltin(*this, E, Intrinsic::fma));
2117 break;
2118
2119 case Builtin::BI__builtin_signbit:
2120 case Builtin::BI__builtin_signbitf:
2121 case Builtin::BI__builtin_signbitl: {
2122 return RValue::get(
2123 Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))),
2124 ConvertType(E->getType())));
2125 }
2126 case Builtin::BI__annotation: {
2127 // Re-encode each wide string to UTF8 and make an MDString.
2128 SmallVector<Metadata *, 1> Strings;
2129 for (const Expr *Arg : E->arguments()) {
2130 const auto *Str = cast<StringLiteral>(Arg->IgnoreParenCasts());
2131 assert(Str->getCharByteWidth() == 2)(static_cast <bool> (Str->getCharByteWidth() == 2) ?
void (0) : __assert_fail ("Str->getCharByteWidth() == 2",
"/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 2131, __extension__ __PRETTY_FUNCTION__))
;
2132 StringRef WideBytes = Str->getBytes();
2133 std::string StrUtf8;
2134 if (!convertUTF16ToUTF8String(
2135 makeArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
2136 CGM.ErrorUnsupported(E, "non-UTF16 __annotation argument");
2137 continue;
2138 }
2139 Strings.push_back(llvm::MDString::get(getLLVMContext(), StrUtf8));
2140 }
2141
2142 // Build and MDTuple of MDStrings and emit the intrinsic call.
2143 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::codeview_annotation, {});
2144 MDTuple *StrTuple = MDTuple::get(getLLVMContext(), Strings);
2145 Builder.CreateCall(F, MetadataAsValue::get(getLLVMContext(), StrTuple));
2146 return RValue::getIgnored();
2147 }
2148 case Builtin::BI__builtin_annotation: {
2149 llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
2150 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
2151 AnnVal->getType());
2152
2153 // Get the annotation string, go through casts. Sema requires this to be a
2154 // non-wide string literal, potentially casted, so the cast<> is safe.
2155 const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
2156 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
2157 return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
2158 }
2159 case Builtin::BI__builtin_addcb:
2160 case Builtin::BI__builtin_addcs:
2161 case Builtin::BI__builtin_addc:
2162 case Builtin::BI__builtin_addcl:
2163 case Builtin::BI__builtin_addcll:
2164 case Builtin::BI__builtin_subcb:
2165 case Builtin::BI__builtin_subcs:
2166 case Builtin::BI__builtin_subc:
2167 case Builtin::BI__builtin_subcl:
2168 case Builtin::BI__builtin_subcll: {
2169
2170 // We translate all of these builtins from expressions of the form:
2171 // int x = ..., y = ..., carryin = ..., carryout, result;
2172 // result = __builtin_addc(x, y, carryin, &carryout);
2173 //
2174 // to LLVM IR of the form:
2175 //
2176 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
2177 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
2178 // %carry1 = extractvalue {i32, i1} %tmp1, 1
2179 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
2180 // i32 %carryin)
2181 // %result = extractvalue {i32, i1} %tmp2, 0
2182 // %carry2 = extractvalue {i32, i1} %tmp2, 1
2183 // %tmp3 = or i1 %carry1, %carry2
2184 // %tmp4 = zext i1 %tmp3 to i32
2185 // store i32 %tmp4, i32* %carryout
2186
2187 // Scalarize our inputs.
2188 llvm::Value *X = EmitScalarExpr(E->getArg(0));
2189 llvm::Value *Y = EmitScalarExpr(E->getArg(1));
2190 llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
2191 Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3));
2192
2193 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
2194 llvm::Intrinsic::ID IntrinsicId;
2195 switch (BuiltinID) {
2196 default: llvm_unreachable("Unknown multiprecision builtin id.")::llvm::llvm_unreachable_internal("Unknown multiprecision builtin id."
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 2196)
;
2197 case Builtin::BI__builtin_addcb:
2198 case Builtin::BI__builtin_addcs:
2199 case Builtin::BI__builtin_addc:
2200 case Builtin::BI__builtin_addcl:
2201 case Builtin::BI__builtin_addcll:
2202 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2203 break;
2204 case Builtin::BI__builtin_subcb:
2205 case Builtin::BI__builtin_subcs:
2206 case Builtin::BI__builtin_subc:
2207 case Builtin::BI__builtin_subcl:
2208 case Builtin::BI__builtin_subcll:
2209 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2210 break;
2211 }
2212
2213 // Construct our resulting LLVM IR expression.
2214 llvm::Value *Carry1;
2215 llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId,
2216 X, Y, Carry1);
2217 llvm::Value *Carry2;
2218 llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId,
2219 Sum1, Carryin, Carry2);
2220 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
2221 X->getType());
2222 Builder.CreateStore(CarryOut, CarryOutPtr);
2223 return RValue::get(Sum2);
2224 }
2225
2226 case Builtin::BI__builtin_add_overflow:
2227 case Builtin::BI__builtin_sub_overflow:
2228 case Builtin::BI__builtin_mul_overflow: {
2229 const clang::Expr *LeftArg = E->getArg(0);
2230 const clang::Expr *RightArg = E->getArg(1);
2231 const clang::Expr *ResultArg = E->getArg(2);
2232
2233 clang::QualType ResultQTy =
2234 ResultArg->getType()->castAs<PointerType>()->getPointeeType();
2235
2236 WidthAndSignedness LeftInfo =
2237 getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType());
2238 WidthAndSignedness RightInfo =
2239 getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType());
2240 WidthAndSignedness ResultInfo =
2241 getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy);
2242 WidthAndSignedness EncompassingInfo =
2243 EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo});
2244
2245 llvm::Type *EncompassingLLVMTy =
2246 llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
2247
2248 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
2249
2250 llvm::Intrinsic::ID IntrinsicId;
2251 switch (BuiltinID) {
2252 default:
2253 llvm_unreachable("Unknown overflow builtin id.")::llvm::llvm_unreachable_internal("Unknown overflow builtin id."
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 2253)
;
2254 case Builtin::BI__builtin_add_overflow:
2255 IntrinsicId = EncompassingInfo.Signed
2256 ? llvm::Intrinsic::sadd_with_overflow
2257 : llvm::Intrinsic::uadd_with_overflow;
2258 break;
2259 case Builtin::BI__builtin_sub_overflow:
2260 IntrinsicId = EncompassingInfo.Signed
2261 ? llvm::Intrinsic::ssub_with_overflow
2262 : llvm::Intrinsic::usub_with_overflow;
2263 break;
2264 case Builtin::BI__builtin_mul_overflow:
2265 IntrinsicId = EncompassingInfo.Signed
2266 ? llvm::Intrinsic::smul_with_overflow
2267 : llvm::Intrinsic::umul_with_overflow;
2268 break;
2269 }
2270
2271 llvm::Value *Left = EmitScalarExpr(LeftArg);
2272 llvm::Value *Right = EmitScalarExpr(RightArg);
2273 Address ResultPtr = EmitPointerWithAlignment(ResultArg);
2274
2275 // Extend each operand to the encompassing type.
2276 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
2277 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
2278
2279 // Perform the operation on the extended values.
2280 llvm::Value *Overflow, *Result;
2281 Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow);
2282
2283 if (EncompassingInfo.Width > ResultInfo.Width) {
2284 // The encompassing type is wider than the result type, so we need to
2285 // truncate it.
2286 llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
2287
2288 // To see if the truncation caused an overflow, we will extend
2289 // the result and then compare it to the original result.
2290 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
2291 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
2292 llvm::Value *TruncationOverflow =
2293 Builder.CreateICmpNE(Result, ResultTruncExt);
2294
2295 Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
2296 Result = ResultTrunc;
2297 }
2298
2299 // Finally, store the result using the pointer.
2300 bool isVolatile =
2301 ResultArg->getType()->getPointeeType().isVolatileQualified();
2302 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
2303
2304 return RValue::get(Overflow);
2305 }
2306
2307 case Builtin::BI__builtin_uadd_overflow:
2308 case Builtin::BI__builtin_uaddl_overflow:
2309 case Builtin::BI__builtin_uaddll_overflow:
2310 case Builtin::BI__builtin_usub_overflow:
2311 case Builtin::BI__builtin_usubl_overflow:
2312 case Builtin::BI__builtin_usubll_overflow:
2313 case Builtin::BI__builtin_umul_overflow:
2314 case Builtin::BI__builtin_umull_overflow:
2315 case Builtin::BI__builtin_umulll_overflow:
2316 case Builtin::BI__builtin_sadd_overflow:
2317 case Builtin::BI__builtin_saddl_overflow:
2318 case Builtin::BI__builtin_saddll_overflow:
2319 case Builtin::BI__builtin_ssub_overflow:
2320 case Builtin::BI__builtin_ssubl_overflow:
2321 case Builtin::BI__builtin_ssubll_overflow:
2322 case Builtin::BI__builtin_smul_overflow:
2323 case Builtin::BI__builtin_smull_overflow:
2324 case Builtin::BI__builtin_smulll_overflow: {
2325
2326 // We translate all of these builtins directly to the relevant llvm IR node.
2327
2328 // Scalarize our inputs.
2329 llvm::Value *X = EmitScalarExpr(E->getArg(0));
2330 llvm::Value *Y = EmitScalarExpr(E->getArg(1));
2331 Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2));
2332
2333 // Decide which of the overflow intrinsics we are lowering to:
2334 llvm::Intrinsic::ID IntrinsicId;
2335 switch (BuiltinID) {
2336 default: llvm_unreachable("Unknown overflow builtin id.")::llvm::llvm_unreachable_internal("Unknown overflow builtin id."
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 2336)
;
2337 case Builtin::BI__builtin_uadd_overflow:
2338 case Builtin::BI__builtin_uaddl_overflow:
2339 case Builtin::BI__builtin_uaddll_overflow:
2340 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2341 break;
2342 case Builtin::BI__builtin_usub_overflow:
2343 case Builtin::BI__builtin_usubl_overflow:
2344 case Builtin::BI__builtin_usubll_overflow:
2345 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2346 break;
2347 case Builtin::BI__builtin_umul_overflow:
2348 case Builtin::BI__builtin_umull_overflow:
2349 case Builtin::BI__builtin_umulll_overflow:
2350 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
2351 break;
2352 case Builtin::BI__builtin_sadd_overflow:
2353 case Builtin::BI__builtin_saddl_overflow:
2354 case Builtin::BI__builtin_saddll_overflow:
2355 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
2356 break;
2357 case Builtin::BI__builtin_ssub_overflow:
2358 case Builtin::BI__builtin_ssubl_overflow:
2359 case Builtin::BI__builtin_ssubll_overflow:
2360 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
2361 break;
2362 case Builtin::BI__builtin_smul_overflow:
2363 case Builtin::BI__builtin_smull_overflow:
2364 case Builtin::BI__builtin_smulll_overflow:
2365 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
2366 break;
2367 }
2368
2369
2370 llvm::Value *Carry;
2371 llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry);
2372 Builder.CreateStore(Sum, SumOutPtr);
2373
2374 return RValue::get(Carry);
2375 }
2376 case Builtin::BI__builtin_addressof:
2377 return RValue::get(EmitLValue(E->getArg(0)).getPointer());
2378 case Builtin::BI__builtin_operator_new:
2379 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(),
2380 E->getArg(0), false);
2381 case Builtin::BI__builtin_operator_delete:
2382 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(),
2383 E->getArg(0), true);
2384 case Builtin::BI__noop:
2385 // __noop always evaluates to an integer literal zero.
2386 return RValue::get(ConstantInt::get(IntTy, 0));
2387 case Builtin::BI__builtin_call_with_static_chain: {
2388 const CallExpr *Call = cast<CallExpr>(E->getArg(0));
2389 const Expr *Chain = E->getArg(1);
2390 return EmitCall(Call->getCallee()->getType(),
2391 EmitCallee(Call->getCallee()), Call, ReturnValue,
2392 EmitScalarExpr(Chain));
2393 }
2394 case Builtin::BI_InterlockedExchange8:
2395 case Builtin::BI_InterlockedExchange16:
2396 case Builtin::BI_InterlockedExchange:
2397 case Builtin::BI_InterlockedExchangePointer:
2398 return RValue::get(
2399 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
2400 case Builtin::BI_InterlockedCompareExchangePointer: {
2401 llvm::Type *RTy;
2402 llvm::IntegerType *IntType =
2403 IntegerType::get(getLLVMContext(),
2404 getContext().getTypeSize(E->getType()));
2405 llvm::Type *IntPtrType = IntType->getPointerTo();
2406
2407 llvm::Value *Destination =
2408 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType);
2409
2410 llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
2411 RTy = Exchange->getType();
2412 Exchange = Builder.CreatePtrToInt(Exchange, IntType);
2413
2414 llvm::Value *Comparand =
2415 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
2416
2417 auto Result =
2418 Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
2419 AtomicOrdering::SequentiallyConsistent,
2420 AtomicOrdering::SequentiallyConsistent);
2421 Result->setVolatile(true);
2422
2423 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
2424 0),
2425 RTy));
2426 }
2427 case Builtin::BI_InterlockedCompareExchange8:
2428 case Builtin::BI_InterlockedCompareExchange16:
2429 case Builtin::BI_InterlockedCompareExchange:
2430 case Builtin::BI_InterlockedCompareExchange64: {
2431 AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg(
2432 EmitScalarExpr(E->getArg(0)),
2433 EmitScalarExpr(E->getArg(2)),
2434 EmitScalarExpr(E->getArg(1)),
2435 AtomicOrdering::SequentiallyConsistent,
2436 AtomicOrdering::SequentiallyConsistent);
2437 CXI->setVolatile(true);
2438 return RValue::get(Builder.CreateExtractValue(CXI, 0));
2439 }
2440 case Builtin::BI_InterlockedIncrement16:
2441 case Builtin::BI_InterlockedIncrement:
2442 return RValue::get(
2443 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E));
2444 case Builtin::BI_InterlockedDecrement16:
2445 case Builtin::BI_InterlockedDecrement:
2446 return RValue::get(
2447 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E));
2448 case Builtin::BI_InterlockedAnd8:
2449 case Builtin::BI_InterlockedAnd16:
2450 case Builtin::BI_InterlockedAnd:
2451 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E));
2452 case Builtin::BI_InterlockedExchangeAdd8:
2453 case Builtin::BI_InterlockedExchangeAdd16:
2454 case Builtin::BI_InterlockedExchangeAdd:
2455 return RValue::get(
2456 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E));
2457 case Builtin::BI_InterlockedExchangeSub8:
2458 case Builtin::BI_InterlockedExchangeSub16:
2459 case Builtin::BI_InterlockedExchangeSub:
2460 return RValue::get(
2461 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E));
2462 case Builtin::BI_InterlockedOr8:
2463 case Builtin::BI_InterlockedOr16:
2464 case Builtin::BI_InterlockedOr:
2465 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E));
2466 case Builtin::BI_InterlockedXor8:
2467 case Builtin::BI_InterlockedXor16:
2468 case Builtin::BI_InterlockedXor:
2469 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E));
2470 case Builtin::BI_interlockedbittestandset:
2471 return RValue::get(
2472 EmitMSVCBuiltinExpr(MSVCIntrin::_interlockedbittestandset, E));
2473
2474 case Builtin::BI__exception_code:
2475 case Builtin::BI_exception_code:
2476 return RValue::get(EmitSEHExceptionCode());
2477 case Builtin::BI__exception_info:
2478 case Builtin::BI_exception_info:
2479 return RValue::get(EmitSEHExceptionInfo());
2480 case Builtin::BI__abnormal_termination:
2481 case Builtin::BI_abnormal_termination:
2482 return RValue::get(EmitSEHAbnormalTermination());
2483 case Builtin::BI_setjmpex: {
2484 if (getTarget().getTriple().isOSMSVCRT()) {
2485 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2486 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
2487 getLLVMContext(), llvm::AttributeList::FunctionIndex,
2488 llvm::Attribute::ReturnsTwice);
2489 llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction(
2490 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false),
2491 "_setjmpex", ReturnsTwiceAttr, /*Local=*/true);
2492 llvm::Value *Buf = Builder.CreateBitOrPointerCast(
2493 EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2494 llvm::Value *FrameAddr =
2495 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2496 ConstantInt::get(Int32Ty, 0));
2497 llvm::Value *Args[] = {Buf, FrameAddr};
2498 llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args);
2499 CS.setAttributes(ReturnsTwiceAttr);
2500 return RValue::get(CS.getInstruction());
2501 }
2502 break;
2503 }
2504 case Builtin::BI_setjmp: {
2505 if (getTarget().getTriple().isOSMSVCRT()) {
2506 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
2507 getLLVMContext(), llvm::AttributeList::FunctionIndex,
2508 llvm::Attribute::ReturnsTwice);
2509 llvm::Value *Buf = Builder.CreateBitOrPointerCast(
2510 EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2511 llvm::CallSite CS;
2512 if (getTarget().getTriple().getArch() == llvm::Triple::x86) {
2513 llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy};
2514 llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction(
2515 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true),
2516 "_setjmp3", ReturnsTwiceAttr, /*Local=*/true);
2517 llvm::Value *Count = ConstantInt::get(IntTy, 0);
2518 llvm::Value *Args[] = {Buf, Count};
2519 CS = EmitRuntimeCallOrInvoke(SetJmp3, Args);
2520 } else {
2521 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2522 llvm::Constant *SetJmp = CGM.CreateRuntimeFunction(
2523 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false),
2524 "_setjmp", ReturnsTwiceAttr, /*Local=*/true);
2525 llvm::Value *FrameAddr =
2526 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2527 ConstantInt::get(Int32Ty, 0));
2528 llvm::Value *Args[] = {Buf, FrameAddr};
2529 CS = EmitRuntimeCallOrInvoke(SetJmp, Args);
2530 }
2531 CS.setAttributes(ReturnsTwiceAttr);
2532 return RValue::get(CS.getInstruction());
2533 }
2534 break;
2535 }
2536
2537 case Builtin::BI__GetExceptionInfo: {
2538 if (llvm::GlobalVariable *GV =
2539 CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
2540 return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
2541 break;
2542 }
2543
2544 case Builtin::BI__fastfail:
2545 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::__fastfail, E));
2546
2547 case Builtin::BI__builtin_coro_size: {
2548 auto & Context = getContext();
2549 auto SizeTy = Context.getSizeType();
2550 auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
2551 Value *F = CGM.getIntrinsic(Intrinsic::coro_size, T);
2552 return RValue::get(Builder.CreateCall(F));
2553 }
2554
2555 case Builtin::BI__builtin_coro_id:
2556 return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
2557 case Builtin::BI__builtin_coro_promise:
2558 return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
2559 case Builtin::BI__builtin_coro_resume:
2560 return EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
2561 case Builtin::BI__builtin_coro_frame:
2562 return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
2563 case Builtin::BI__builtin_coro_free:
2564 return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
2565 case Builtin::BI__builtin_coro_destroy:
2566 return EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
2567 case Builtin::BI__builtin_coro_done:
2568 return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
2569 case Builtin::BI__builtin_coro_alloc:
2570 return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
2571 case Builtin::BI__builtin_coro_begin:
2572 return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
2573 case Builtin::BI__builtin_coro_end:
2574 return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
2575 case Builtin::BI__builtin_coro_suspend:
2576 return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
2577 case Builtin::BI__builtin_coro_param:
2578 return EmitCoroutineIntrinsic(E, Intrinsic::coro_param);
2579
2580 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
2581 case Builtin::BIread_pipe:
2582 case Builtin::BIwrite_pipe: {
2583 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2584 *Arg1 = EmitScalarExpr(E->getArg(1));
2585 CGOpenCLRuntime OpenCLRT(CGM);
2586 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2587 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2588
2589 // Type of the generic packet parameter.
2590 unsigned GenericAS =
2591 getContext().getTargetAddressSpace(LangAS::opencl_generic);
2592 llvm::Type *I8PTy = llvm::PointerType::get(
2593 llvm::Type::getInt8Ty(getLLVMContext()), GenericAS);
2594
2595 // Testing which overloaded version we should generate the call for.
2596 if (2U == E->getNumArgs()) {
2597 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
2598 : "__write_pipe_2";
2599 // Creating a generic function type to be able to call with any builtin or
2600 // user defined type.
2601 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
2602 llvm::FunctionType *FTy = llvm::FunctionType::get(
2603 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2604 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
2605 return RValue::get(
2606 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2607 {Arg0, BCast, PacketSize, PacketAlign}));
2608 } else {
2609 assert(4 == E->getNumArgs() &&(static_cast <bool> (4 == E->getNumArgs() &&
"Illegal number of parameters to pipe function") ? void (0) :
__assert_fail ("4 == E->getNumArgs() && \"Illegal number of parameters to pipe function\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 2610, __extension__ __PRETTY_FUNCTION__))
2610 "Illegal number of parameters to pipe function")(static_cast <bool> (4 == E->getNumArgs() &&
"Illegal number of parameters to pipe function") ? void (0) :
__assert_fail ("4 == E->getNumArgs() && \"Illegal number of parameters to pipe function\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 2610, __extension__ __PRETTY_FUNCTION__))
;
2611 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
2612 : "__write_pipe_4";
2613
2614 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
2615 Int32Ty, Int32Ty};
2616 Value *Arg2 = EmitScalarExpr(E->getArg(2)),
2617 *Arg3 = EmitScalarExpr(E->getArg(3));
2618 llvm::FunctionType *FTy = llvm::FunctionType::get(
2619 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2620 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
2621 // We know the third argument is an integer type, but we may need to cast
2622 // it to i32.
2623 if (Arg2->getType() != Int32Ty)
2624 Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
2625 return RValue::get(Builder.CreateCall(
2626 CGM.CreateRuntimeFunction(FTy, Name),
2627 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
2628 }
2629 }
2630 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
2631 // functions
2632 case Builtin::BIreserve_read_pipe:
2633 case Builtin::BIreserve_write_pipe:
2634 case Builtin::BIwork_group_reserve_read_pipe:
2635 case Builtin::BIwork_group_reserve_write_pipe:
2636 case Builtin::BIsub_group_reserve_read_pipe:
2637 case Builtin::BIsub_group_reserve_write_pipe: {
2638 // Composing the mangled name for the function.
2639 const char *Name;
2640 if (BuiltinID == Builtin::BIreserve_read_pipe)
2641 Name = "__reserve_read_pipe";
2642 else if (BuiltinID == Builtin::BIreserve_write_pipe)
2643 Name = "__reserve_write_pipe";
2644 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
2645 Name = "__work_group_reserve_read_pipe";
2646 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
2647 Name = "__work_group_reserve_write_pipe";
2648 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
2649 Name = "__sub_group_reserve_read_pipe";
2650 else
2651 Name = "__sub_group_reserve_write_pipe";
2652
2653 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2654 *Arg1 = EmitScalarExpr(E->getArg(1));
2655 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
2656 CGOpenCLRuntime OpenCLRT(CGM);
2657 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2658 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2659
2660 // Building the generic function prototype.
2661 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
2662 llvm::FunctionType *FTy = llvm::FunctionType::get(
2663 ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2664 // We know the second argument is an integer type, but we may need to cast
2665 // it to i32.
2666 if (Arg1->getType() != Int32Ty)
2667 Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
2668 return RValue::get(
2669 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2670 {Arg0, Arg1, PacketSize, PacketAlign}));
2671 }
2672 // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
2673 // functions
2674 case Builtin::BIcommit_read_pipe:
2675 case Builtin::BIcommit_write_pipe:
2676 case Builtin::BIwork_group_commit_read_pipe:
2677 case Builtin::BIwork_group_commit_write_pipe:
2678 case Builtin::BIsub_group_commit_read_pipe:
2679 case Builtin::BIsub_group_commit_write_pipe: {
2680 const char *Name;
2681 if (BuiltinID == Builtin::BIcommit_read_pipe)
2682 Name = "__commit_read_pipe";
2683 else if (BuiltinID == Builtin::BIcommit_write_pipe)
2684 Name = "__commit_write_pipe";
2685 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
2686 Name = "__work_group_commit_read_pipe";
2687 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
2688 Name = "__work_group_commit_write_pipe";
2689 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
2690 Name = "__sub_group_commit_read_pipe";
2691 else
2692 Name = "__sub_group_commit_write_pipe";
2693
2694 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2695 *Arg1 = EmitScalarExpr(E->getArg(1));
2696 CGOpenCLRuntime OpenCLRT(CGM);
2697 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2698 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2699
2700 // Building the generic function prototype.
2701 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
2702 llvm::FunctionType *FTy =
2703 llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
2704 llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2705
2706 return RValue::get(
2707 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2708 {Arg0, Arg1, PacketSize, PacketAlign}));
2709 }
2710 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
2711 case Builtin::BIget_pipe_num_packets:
2712 case Builtin::BIget_pipe_max_packets: {
2713 const char *Name;
2714 if (BuiltinID == Builtin::BIget_pipe_num_packets)
2715 Name = "__get_pipe_num_packets";
2716 else
2717 Name = "__get_pipe_max_packets";
2718
2719 // Building the generic function prototype.
2720 Value *Arg0 = EmitScalarExpr(E->getArg(0));
2721 CGOpenCLRuntime OpenCLRT(CGM);
2722 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2723 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2724 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
2725 llvm::FunctionType *FTy = llvm::FunctionType::get(
2726 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2727
2728 return RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2729 {Arg0, PacketSize, PacketAlign}));
2730 }
2731
2732 // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
2733 case Builtin::BIto_global:
2734 case Builtin::BIto_local:
2735 case Builtin::BIto_private: {
2736 auto Arg0 = EmitScalarExpr(E->getArg(0));
2737 auto NewArgT = llvm::PointerType::get(Int8Ty,
2738 CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
2739 auto NewRetT = llvm::PointerType::get(Int8Ty,
2740 CGM.getContext().getTargetAddressSpace(
2741 E->getType()->getPointeeType().getAddressSpace()));
2742 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false);
2743 llvm::Value *NewArg;
2744 if (Arg0->getType()->getPointerAddressSpace() !=
2745 NewArgT->getPointerAddressSpace())
2746 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
2747 else
2748 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
2749 auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
2750 auto NewCall =
2751 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
2752 return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
2753 ConvertType(E->getType())));
2754 }
2755
2756 // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
2757 // It contains four different overload formats specified in Table 6.13.17.1.
2758 case Builtin::BIenqueue_kernel: {
2759 StringRef Name; // Generated function call name
2760 unsigned NumArgs = E->getNumArgs();
2761
2762 llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
2763 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
2764 getContext().getTargetAddressSpace(LangAS::opencl_generic));
2765
2766 llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
2767 llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
2768 LValue NDRangeL = EmitAggExprToLValue(E->getArg(2));
2769 llvm::Value *Range = NDRangeL.getAddress().getPointer();
2770 llvm::Type *RangeTy = NDRangeL.getAddress().getType();
2771
2772 if (NumArgs == 4) {
2773 // The most basic form of the call with parameters:
2774 // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
2775 Name = "__enqueue_kernel_basic";
2776 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, GenericVoidPtrTy,
2777 GenericVoidPtrTy};
2778 llvm::FunctionType *FTy = llvm::FunctionType::get(
2779 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2780
2781 auto Info =
2782 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
2783 llvm::Value *Kernel =
2784 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
2785 llvm::Value *Block =
2786 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
2787
2788 AttrBuilder B;
2789 B.addAttribute(Attribute::ByVal);
2790 llvm::AttributeList ByValAttrSet =
2791 llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B);
2792
2793 auto RTCall =
2794 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
2795 {Queue, Flags, Range, Kernel, Block});
2796 RTCall->setAttributes(ByValAttrSet);
2797 return RValue::get(RTCall);
2798 }
2799 assert(NumArgs >= 5 && "Invalid enqueue_kernel signature")(static_cast <bool> (NumArgs >= 5 && "Invalid enqueue_kernel signature"
) ? void (0) : __assert_fail ("NumArgs >= 5 && \"Invalid enqueue_kernel signature\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 2799, __extension__ __PRETTY_FUNCTION__))
;
2800
2801 // Create a temporary array to hold the sizes of local pointer arguments
2802 // for the block. \p First is the position of the first size argument.
2803 auto CreateArrayForSizeVar = [=](unsigned First) {
2804 auto *AT = llvm::ArrayType::get(SizeTy, NumArgs - First);
2805 auto *Arr = Builder.CreateAlloca(AT);
2806 llvm::Value *Ptr;
2807 // Each of the following arguments specifies the size of the corresponding
2808 // argument passed to the enqueued block.
2809 auto *Zero = llvm::ConstantInt::get(IntTy, 0);
2810 for (unsigned I = First; I < NumArgs; ++I) {
2811 auto *Index = llvm::ConstantInt::get(IntTy, I - First);
2812 auto *GEP = Builder.CreateGEP(Arr, {Zero, Index});
2813 if (I == First)
2814 Ptr = GEP;
2815 auto *V =
2816 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy);
2817 Builder.CreateAlignedStore(
2818 V, GEP, CGM.getDataLayout().getPrefTypeAlignment(SizeTy));
2819 }
2820 return Ptr;
2821 };
2822
2823 // Could have events and/or vaargs.
2824 if (E->getArg(3)->getType()->isBlockPointerType()) {
2825 // No events passed, but has variadic arguments.
2826 Name = "__enqueue_kernel_vaargs";
2827 auto Info =
2828 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
2829 llvm::Value *Kernel =
2830 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
2831 auto *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
2832 auto *PtrToSizeArray = CreateArrayForSizeVar(4);
2833
2834 // Create a vector of the arguments, as well as a constant value to
2835 // express to the runtime the number of variadic arguments.
2836 std::vector<llvm::Value *> Args = {
2837 Queue, Flags, Range,
2838 Kernel, Block, ConstantInt::get(IntTy, NumArgs - 4),
2839 PtrToSizeArray};
2840 std::vector<llvm::Type *> ArgTys = {
2841 QueueTy, IntTy, RangeTy,
2842 GenericVoidPtrTy, GenericVoidPtrTy, IntTy,
2843 PtrToSizeArray->getType()};
2844
2845 llvm::FunctionType *FTy = llvm::FunctionType::get(
2846 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2847 return RValue::get(
2848 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2849 llvm::ArrayRef<llvm::Value *>(Args)));
2850 }
2851 // Any calls now have event arguments passed.
2852 if (NumArgs >= 7) {
2853 llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy);
2854 llvm::Type *EventPtrTy = EventTy->getPointerTo(
2855 CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
2856
2857 llvm::Value *NumEvents =
2858 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(3)), Int32Ty);
2859 llvm::Value *EventList =
2860 E->getArg(4)->getType()->isArrayType()
2861 ? EmitArrayToPointerDecay(E->getArg(4)).getPointer()
2862 : EmitScalarExpr(E->getArg(4));
2863 llvm::Value *ClkEvent = EmitScalarExpr(E->getArg(5));
2864 // Convert to generic address space.
2865 EventList = Builder.CreatePointerCast(EventList, EventPtrTy);
2866 ClkEvent = Builder.CreatePointerCast(ClkEvent, EventPtrTy);
2867 auto Info =
2868 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(6));
2869 llvm::Value *Kernel =
2870 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
2871 llvm::Value *Block =
2872 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
2873
2874 std::vector<llvm::Type *> ArgTys = {
2875 QueueTy, Int32Ty, RangeTy, Int32Ty,
2876 EventPtrTy, EventPtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
2877
2878 std::vector<llvm::Value *> Args = {Queue, Flags, Range, NumEvents,
2879 EventList, ClkEvent, Kernel, Block};
2880
2881 if (NumArgs == 7) {
2882 // Has events but no variadics.
2883 Name = "__enqueue_kernel_basic_events";
2884 llvm::FunctionType *FTy = llvm::FunctionType::get(
2885 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2886 return RValue::get(
2887 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2888 llvm::ArrayRef<llvm::Value *>(Args)));
2889 }
2890 // Has event info and variadics
2891 // Pass the number of variadics to the runtime function too.
2892 Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
2893 ArgTys.push_back(Int32Ty);
2894 Name = "__enqueue_kernel_events_vaargs";
2895
2896 auto *PtrToSizeArray = CreateArrayForSizeVar(7);
2897 Args.push_back(PtrToSizeArray);
2898 ArgTys.push_back(PtrToSizeArray->getType());
2899
2900 llvm::FunctionType *FTy = llvm::FunctionType::get(
2901 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2902 return RValue::get(
2903 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2904 llvm::ArrayRef<llvm::Value *>(Args)));
2905 }
2906 LLVM_FALLTHROUGH[[clang::fallthrough]];
2907 }
2908 // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
2909 // parameter.
2910 case Builtin::BIget_kernel_work_group_size: {
2911 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
2912 getContext().getTargetAddressSpace(LangAS::opencl_generic));
2913 auto Info =
2914 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
2915 Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
2916 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
2917 return RValue::get(Builder.CreateCall(
2918 CGM.CreateRuntimeFunction(
2919 llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
2920 false),
2921 "__get_kernel_work_group_size_impl"),
2922 {Kernel, Arg}));
2923 }
2924 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
2925 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
2926 getContext().getTargetAddressSpace(LangAS::opencl_generic));
2927 auto Info =
2928 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
2929 Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
2930 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
2931 return RValue::get(Builder.CreateCall(
2932 CGM.CreateRuntimeFunction(
2933 llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
2934 false),
2935 "__get_kernel_preferred_work_group_multiple_impl"),
2936 {Kernel, Arg}));
2937 }
2938 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
2939 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
2940 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
2941 getContext().getTargetAddressSpace(LangAS::opencl_generic));
2942 LValue NDRangeL = EmitAggExprToLValue(E->getArg(0));
2943 llvm::Value *NDRange = NDRangeL.getAddress().getPointer();
2944 auto Info =
2945 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1));
2946 Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
2947 Value *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
2948 const char *Name =
2949 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
2950 ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
2951 : "__get_kernel_sub_group_count_for_ndrange_impl";
2952 return RValue::get(Builder.CreateCall(
2953 CGM.CreateRuntimeFunction(
2954 llvm::FunctionType::get(
2955 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
2956 false),
2957 Name),
2958 {NDRange, Kernel, Block}));
2959 }
2960
2961 case Builtin::BI__builtin_store_half:
2962 case Builtin::BI__builtin_store_halff: {
2963 Value *Val = EmitScalarExpr(E->getArg(0));
2964 Address Address = EmitPointerWithAlignment(E->getArg(1));
2965 Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
2966 return RValue::get(Builder.CreateStore(HalfVal, Address));
2967 }
2968 case Builtin::BI__builtin_load_half: {
2969 Address Address = EmitPointerWithAlignment(E->getArg(0));
2970 Value *HalfVal = Builder.CreateLoad(Address);
2971 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
2972 }
2973 case Builtin::BI__builtin_load_halff: {
2974 Address Address = EmitPointerWithAlignment(E->getArg(0));
2975 Value *HalfVal = Builder.CreateLoad(Address);
2976 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
2977 }
2978 case Builtin::BIprintf:
2979 if (getTarget().getTriple().isNVPTX())
2980 return EmitNVPTXDevicePrintfCallExpr(E, ReturnValue);
2981 break;
2982 case Builtin::BI__builtin_canonicalize:
2983 case Builtin::BI__builtin_canonicalizef:
2984 case Builtin::BI__builtin_canonicalizel:
2985 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize));
2986
2987 case Builtin::BI__builtin_thread_pointer: {
2988 if (!getContext().getTargetInfo().isTLSSupported())
2989 CGM.ErrorUnsupported(E, "__builtin_thread_pointer");
2990 // Fall through - it's already mapped to the intrinsic by GCCBuiltin.
2991 break;
2992 }
2993 case Builtin::BI__builtin_os_log_format:
2994 return emitBuiltinOSLogFormat(*E);
2995
2996 case Builtin::BI__builtin_os_log_format_buffer_size: {
2997 analyze_os_log::OSLogBufferLayout Layout;
2998 analyze_os_log::computeOSLogBufferLayout(CGM.getContext(), E, Layout);
2999 return RValue::get(ConstantInt::get(ConvertType(E->getType()),
3000 Layout.size().getQuantity()));
3001 }
3002
3003 case Builtin::BI__xray_customevent: {
3004 if (!ShouldXRayInstrumentFunction())
3005 return RValue::getIgnored();
3006 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>()) {
3007 if (XRayAttr->neverXRayInstrument())
3008 return RValue::getIgnored();
3009 }
3010 Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
3011 auto FTy = F->getFunctionType();
3012 auto Arg0 = E->getArg(0);
3013 auto Arg0Val = EmitScalarExpr(Arg0);
3014 auto Arg0Ty = Arg0->getType();
3015 auto PTy0 = FTy->getParamType(0);
3016 if (PTy0 != Arg0Val->getType()) {
3017 if (Arg0Ty->isArrayType())
3018 Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer();
3019 else
3020 Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
3021 }
3022 auto Arg1 = EmitScalarExpr(E->getArg(1));
3023 auto PTy1 = FTy->getParamType(1);
3024 if (PTy1 != Arg1->getType())
3025 Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
3026 return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
3027 }
3028
3029 case Builtin::BI__builtin_ms_va_start:
3030 case Builtin::BI__builtin_ms_va_end:
3031 return RValue::get(
3032 EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
3033 BuiltinID == Builtin::BI__builtin_ms_va_start));
3034
3035 case Builtin::BI__builtin_ms_va_copy: {
3036 // Lower this manually. We can't reliably determine whether or not any
3037 // given va_copy() is for a Win64 va_list from the calling convention
3038 // alone, because it's legal to do this from a System V ABI function.
3039 // With opaque pointer types, we won't have enough information in LLVM
3040 // IR to determine this from the argument types, either. Best to do it
3041 // now, while we have enough information.
3042 Address DestAddr = EmitMSVAListRef(E->getArg(0));
3043 Address SrcAddr = EmitMSVAListRef(E->getArg(1));
3044
3045 llvm::Type *BPP = Int8PtrPtrTy;
3046
3047 DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"),
3048 DestAddr.getAlignment());
3049 SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"),
3050 SrcAddr.getAlignment());
3051
3052 Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val");
3053 return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
3054 }
3055 }
3056
3057 // If this is an alias for a lib function (e.g. __builtin_sin), emit
3058 // the call using the normal call path, but using the unmangled
3059 // version of the function name.
3060 if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
3061 return emitLibraryCall(*this, FD, E,
3062 CGM.getBuiltinLibFunction(FD, BuiltinID));
3063
3064 // If this is a predefined lib function (e.g. malloc), emit the call
3065 // using exactly the normal call path.
3066 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
3067 return emitLibraryCall(*this, FD, E,
3068 cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
3069
3070 // Check that a call to a target specific builtin has the correct target
3071 // features.
3072 // This is down here to avoid non-target specific builtins, however, if
3073 // generic builtins start to require generic target features then we
3074 // can move this up to the beginning of the function.
3075 checkTargetFeatures(E, FD);
3076
3077 // See if we have a target specific intrinsic.
3078 const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
3079 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
3080 StringRef Prefix =
3081 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
3082 if (!Prefix.empty()) {
3083 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(), Name);
3084 // NOTE we dont need to perform a compatibility flag check here since the
3085 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
3086 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
3087 if (IntrinsicID == Intrinsic::not_intrinsic)
3088 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
3089 }
3090
3091 if (IntrinsicID != Intrinsic::not_intrinsic) {
3092 SmallVector<Value*, 16> Args;
3093
3094 // Find out if any arguments are required to be integer constant
3095 // expressions.
3096 unsigned ICEArguments = 0;
3097 ASTContext::GetBuiltinTypeError Error;
3098 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
3099 assert(Error == ASTContext::GE_None && "Should not codegen an error")(static_cast <bool> (Error == ASTContext::GE_None &&
"Should not codegen an error") ? void (0) : __assert_fail ("Error == ASTContext::GE_None && \"Should not codegen an error\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3099, __extension__ __PRETTY_FUNCTION__))
;
3100
3101 Function *F = CGM.getIntrinsic(IntrinsicID);
3102 llvm::FunctionType *FTy = F->getFunctionType();
3103
3104 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
3105 Value *ArgValue;
3106 // If this is a normal argument, just emit it as a scalar.
3107 if ((ICEArguments & (1 << i)) == 0) {
3108 ArgValue = EmitScalarExpr(E->getArg(i));
3109 } else {
3110 // If this is required to be a constant, constant fold it so that we
3111 // know that the generated intrinsic gets a ConstantInt.
3112 llvm::APSInt Result;
3113 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
3114 assert(IsConst && "Constant arg isn't actually constant?")(static_cast <bool> (IsConst && "Constant arg isn't actually constant?"
) ? void (0) : __assert_fail ("IsConst && \"Constant arg isn't actually constant?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3114, __extension__ __PRETTY_FUNCTION__))
;
3115 (void)IsConst;
3116 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
3117 }
3118
3119 // If the intrinsic arg type is different from the builtin arg type
3120 // we need to do a bit cast.
3121 llvm::Type *PTy = FTy->getParamType(i);
3122 if (PTy != ArgValue->getType()) {
3123 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&(static_cast <bool> (PTy->canLosslesslyBitCastTo(FTy
->getParamType(i)) && "Must be able to losslessly bit cast to param"
) ? void (0) : __assert_fail ("PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && \"Must be able to losslessly bit cast to param\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3124, __extension__ __PRETTY_FUNCTION__))
3124 "Must be able to losslessly bit cast to param")(static_cast <bool> (PTy->canLosslesslyBitCastTo(FTy
->getParamType(i)) && "Must be able to losslessly bit cast to param"
) ? void (0) : __assert_fail ("PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && \"Must be able to losslessly bit cast to param\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3124, __extension__ __PRETTY_FUNCTION__))
;
3125 ArgValue = Builder.CreateBitCast(ArgValue, PTy);
3126 }
3127
3128 Args.push_back(ArgValue);
3129 }
3130
3131 Value *V = Builder.CreateCall(F, Args);
3132 QualType BuiltinRetType = E->getType();
3133
3134 llvm::Type *RetTy = VoidTy;
3135 if (!BuiltinRetType->isVoidType())
3136 RetTy = ConvertType(BuiltinRetType);
3137
3138 if (RetTy != V->getType()) {
3139 assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&(static_cast <bool> (V->getType()->canLosslesslyBitCastTo
(RetTy) && "Must be able to losslessly bit cast result type"
) ? void (0) : __assert_fail ("V->getType()->canLosslesslyBitCastTo(RetTy) && \"Must be able to losslessly bit cast result type\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3140, __extension__ __PRETTY_FUNCTION__))
3140 "Must be able to losslessly bit cast result type")(static_cast <bool> (V->getType()->canLosslesslyBitCastTo
(RetTy) && "Must be able to losslessly bit cast result type"
) ? void (0) : __assert_fail ("V->getType()->canLosslesslyBitCastTo(RetTy) && \"Must be able to losslessly bit cast result type\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3140, __extension__ __PRETTY_FUNCTION__))
;
3141 V = Builder.CreateBitCast(V, RetTy);
3142 }
3143
3144 return RValue::get(V);
3145 }
3146
3147 // See if we have a target specific builtin that needs to be lowered.
3148 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
3149 return RValue::get(V);
3150
3151 ErrorUnsupported(E, "builtin function");
3152
3153 // Unknown builtin, for now just dump it out and return undef.
3154 return GetUndefRValue(E->getType());
3155}
3156
3157static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
3158 unsigned BuiltinID, const CallExpr *E,
3159 llvm::Triple::ArchType Arch) {
3160 switch (Arch) {
3161 case llvm::Triple::arm:
3162 case llvm::Triple::armeb:
3163 case llvm::Triple::thumb:
3164 case llvm::Triple::thumbeb:
3165 return CGF->EmitARMBuiltinExpr(BuiltinID, E);
3166 case llvm::Triple::aarch64:
3167 case llvm::Triple::aarch64_be:
3168 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E);
3169 case llvm::Triple::x86:
3170 case llvm::Triple::x86_64:
3171 return CGF->EmitX86BuiltinExpr(BuiltinID, E);
3172 case llvm::Triple::ppc:
3173 case llvm::Triple::ppc64:
3174 case llvm::Triple::ppc64le:
3175 return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
3176 case llvm::Triple::r600:
3177 case llvm::Triple::amdgcn:
3178 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
3179 case llvm::Triple::systemz:
3180 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
3181 case llvm::Triple::nvptx:
3182 case llvm::Triple::nvptx64:
3183 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
3184 case llvm::Triple::wasm32:
3185 case llvm::Triple::wasm64:
3186 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
3187 default:
3188 return nullptr;
3189 }
3190}
3191
3192Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
3193 const CallExpr *E) {
3194 if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
3195 assert(getContext().getAuxTargetInfo() && "Missing aux target info")(static_cast <bool> (getContext().getAuxTargetInfo() &&
"Missing aux target info") ? void (0) : __assert_fail ("getContext().getAuxTargetInfo() && \"Missing aux target info\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3195, __extension__ __PRETTY_FUNCTION__))
;
3196 return EmitTargetArchBuiltinExpr(
3197 this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E,
3198 getContext().getAuxTargetInfo()->getTriple().getArch());
3199 }
3200
3201 return EmitTargetArchBuiltinExpr(this, BuiltinID, E,
3202 getTarget().getTriple().getArch());
3203}
3204
3205static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
3206 NeonTypeFlags TypeFlags,
3207 bool V1Ty=false) {
3208 int IsQuad = TypeFlags.isQuad();
3209 switch (TypeFlags.getEltType()) {
3210 case NeonTypeFlags::Int8:
3211 case NeonTypeFlags::Poly8:
3212 return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
3213 case NeonTypeFlags::Int16:
3214 case NeonTypeFlags::Poly16:
3215 case NeonTypeFlags::Float16:
3216 return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
3217 case NeonTypeFlags::Int32:
3218 return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
3219 case NeonTypeFlags::Int64:
3220 case NeonTypeFlags::Poly64:
3221 return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
3222 case NeonTypeFlags::Poly128:
3223 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
3224 // There is a lot of i128 and f128 API missing.
3225 // so we use v16i8 to represent poly128 and get pattern matched.
3226 return llvm::VectorType::get(CGF->Int8Ty, 16);
3227 case NeonTypeFlags::Float32:
3228 return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
3229 case NeonTypeFlags::Float64:
3230 return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
3231 }
3232 llvm_unreachable("Unknown vector element type!")::llvm::llvm_unreachable_internal("Unknown vector element type!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3232)
;
3233}
3234
3235static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
3236 NeonTypeFlags IntTypeFlags) {
3237 int IsQuad = IntTypeFlags.isQuad();
3238 switch (IntTypeFlags.getEltType()) {
3239 case NeonTypeFlags::Int32:
3240 return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad));
3241 case NeonTypeFlags::Int64:
3242 return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad));
3243 default:
3244 llvm_unreachable("Type can't be converted to floating-point!")::llvm::llvm_unreachable_internal("Type can't be converted to floating-point!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3244)
;
3245 }
3246}
3247
3248Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
3249 unsigned nElts = V->getType()->getVectorNumElements();
3250 Value* SV = llvm::ConstantVector::getSplat(nElts, C);
3251 return Builder.CreateShuffleVector(V, V, SV, "lane");
3252}
3253
3254Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
3255 const char *name,
3256 unsigned shift, bool rightshift) {
3257 unsigned j = 0;
3258 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
3259 ai != ae; ++ai, ++j)
3260 if (shift > 0 && shift == j)
3261 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
3262 else
3263 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
3264
3265 return Builder.CreateCall(F, Ops, name);
3266}
3267
3268Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
3269 bool neg) {
3270 int SV = cast<ConstantInt>(V)->getSExtValue();
3271 return ConstantInt::get(Ty, neg ? -SV : SV);
3272}
3273
3274// \brief Right-shift a vector by a constant.
3275Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
3276 llvm::Type *Ty, bool usgn,
3277 const char *name) {
3278 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
3279
3280 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
3281 int EltSize = VTy->getScalarSizeInBits();
3282
3283 Vec = Builder.CreateBitCast(Vec, Ty);
3284
3285 // lshr/ashr are undefined when the shift amount is equal to the vector
3286 // element size.
3287 if (ShiftAmt == EltSize) {
3288 if (usgn) {
3289 // Right-shifting an unsigned value by its size yields 0.
3290 return llvm::ConstantAggregateZero::get(VTy);
3291 } else {
3292 // Right-shifting a signed value by its size is equivalent
3293 // to a shift of size-1.
3294 --ShiftAmt;
3295 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
3296 }
3297 }
3298
3299 Shift = EmitNeonShiftVector(Shift, Ty, false);
3300 if (usgn)
3301 return Builder.CreateLShr(Vec, Shift, name);
3302 else
3303 return Builder.CreateAShr(Vec, Shift, name);
3304}
3305
3306enum {
3307 AddRetType = (1 << 0),
3308 Add1ArgType = (1 << 1),
3309 Add2ArgTypes = (1 << 2),
3310
3311 VectorizeRetType = (1 << 3),
3312 VectorizeArgTypes = (1 << 4),
3313
3314 InventFloatType = (1 << 5),
3315 UnsignedAlts = (1 << 6),
3316
3317 Use64BitVectors = (1 << 7),
3318 Use128BitVectors = (1 << 8),
3319
3320 Vectorize1ArgType = Add1ArgType | VectorizeArgTypes,
3321 VectorRet = AddRetType | VectorizeRetType,
3322 VectorRetGetArgs01 =
3323 AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes,
3324 FpCmpzModifiers =
3325 AddRetType | VectorizeRetType | Add1ArgType | InventFloatType
3326};
3327
3328namespace {
3329struct NeonIntrinsicInfo {
3330 const char *NameHint;
3331 unsigned BuiltinID;
3332 unsigned LLVMIntrinsic;
3333 unsigned AltLLVMIntrinsic;
3334 unsigned TypeModifier;
3335
3336 bool operator<(unsigned RHSBuiltinID) const {
3337 return BuiltinID < RHSBuiltinID;
3338 }
3339 bool operator<(const NeonIntrinsicInfo &TE) const {
3340 return BuiltinID < TE.BuiltinID;
3341 }
3342};
3343} // end anonymous namespace
3344
3345#define NEONMAP0(NameBase) \
3346 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
3347
3348#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
3349 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
3350 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
3351
3352#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
3353 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
3354 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
3355 TypeModifier }
3356
3357static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = {
3358 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
3359 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
3360 NEONMAP1(vabs_v, arm_neon_vabs, 0),
3361 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
3362 NEONMAP0(vaddhn_v),
3363 NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
3364 NEONMAP1(vaeseq_v, arm_neon_aese, 0),
3365 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
3366 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
3367 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
3368 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
3369 NEONMAP1(vcage_v, arm_neon_vacge, 0),
3370 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
3371 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
3372 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
3373 NEONMAP1(vcale_v, arm_neon_vacge, 0),
3374 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
3375 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
3376 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
3377 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
3378 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
3379 NEONMAP1(vclz_v, ctlz, Add1ArgType),
3380 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
3381 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
3382 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
3383 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
3384 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
3385 NEONMAP0(vcvt_f32_v),
3386 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3387 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3388 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3389 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3390 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
3391 NEONMAP0(vcvt_s32_v),
3392 NEONMAP0(vcvt_s64_v),
3393 NEONMAP0(vcvt_u32_v),
3394 NEONMAP0(vcvt_u64_v),
3395 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
3396 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
3397 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
3398 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
3399 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
3400 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
3401 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
3402 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
3403 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
3404 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
3405 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
3406 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
3407 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
3408 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
3409 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
3410 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
3411 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
3412 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
3413 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
3414 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
3415 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
3416 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
3417 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
3418 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
3419 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
3420 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
3421 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
3422 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
3423 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
3424 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
3425 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
3426 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
3427 NEONMAP0(vcvtq_f32_v),
3428 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3429 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3430 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3431 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3432 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
3433 NEONMAP0(vcvtq_s32_v),
3434 NEONMAP0(vcvtq_s64_v),
3435 NEONMAP0(vcvtq_u32_v),
3436 NEONMAP0(vcvtq_u64_v),
3437 NEONMAP0(vext_v),
3438 NEONMAP0(vextq_v),
3439 NEONMAP0(vfma_v),
3440 NEONMAP0(vfmaq_v),
3441 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
3442 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
3443 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
3444 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
3445 NEONMAP0(vld1_dup_v),
3446 NEONMAP1(vld1_v, arm_neon_vld1, 0),
3447 NEONMAP0(vld1q_dup_v),
3448 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
3449 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
3450 NEONMAP1(vld2_v, arm_neon_vld2, 0),
3451 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
3452 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
3453 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
3454 NEONMAP1(vld3_v, arm_neon_vld3, 0),
3455 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
3456 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
3457 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
3458 NEONMAP1(vld4_v, arm_neon_vld4, 0),
3459 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
3460 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
3461 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
3462 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
3463 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
3464 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
3465 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
3466 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
3467 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
3468 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
3469 NEONMAP0(vmovl_v),
3470 NEONMAP0(vmovn_v),
3471 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
3472 NEONMAP0(vmull_v),
3473 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
3474 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
3475 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
3476 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
3477 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
3478 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
3479 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
3480 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
3481 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
3482 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
3483 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
3484 NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
3485 NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
3486 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0),
3487 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0),
3488 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
3489 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
3490 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
3491 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
3492 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
3493 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
3494 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
3495 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
3496 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
3497 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
3498 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
3499 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
3500 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
3501 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
3502 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
3503 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
3504 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
3505 NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
3506 NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
3507 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
3508 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
3509 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
3510 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
3511 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
3512 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
3513 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
3514 NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType),
3515 NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType),
3516 NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType),
3517 NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType),
3518 NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType),
3519 NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType),
3520 NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType),
3521 NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType),
3522 NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType),
3523 NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType),
3524 NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType),
3525 NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType),
3526 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
3527 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
3528 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
3529 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
3530 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
3531 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
3532 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
3533 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
3534 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
3535 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
3536 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
3537 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
3538 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
3539 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
3540 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
3541 NEONMAP0(vshl_n_v),
3542 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
3543 NEONMAP0(vshll_n_v),
3544 NEONMAP0(vshlq_n_v),
3545 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
3546 NEONMAP0(vshr_n_v),
3547 NEONMAP0(vshrn_n_v),
3548 NEONMAP0(vshrq_n_v),
3549 NEONMAP1(vst1_v, arm_neon_vst1, 0),
3550 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
3551 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
3552 NEONMAP1(vst2_v, arm_neon_vst2, 0),
3553 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
3554 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
3555 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
3556 NEONMAP1(vst3_v, arm_neon_vst3, 0),
3557 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
3558 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
3559 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
3560 NEONMAP1(vst4_v, arm_neon_vst4, 0),
3561 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
3562 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
3563 NEONMAP0(vsubhn_v),
3564 NEONMAP0(vtrn_v),
3565 NEONMAP0(vtrnq_v),
3566 NEONMAP0(vtst_v),
3567 NEONMAP0(vtstq_v),
3568 NEONMAP0(vuzp_v),
3569 NEONMAP0(vuzpq_v),
3570 NEONMAP0(vzip_v),
3571 NEONMAP0(vzipq_v)
3572};
3573
3574static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
3575 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
3576 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
3577 NEONMAP0(vaddhn_v),
3578 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
3579 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
3580 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
3581 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
3582 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
3583 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
3584 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
3585 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
3586 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
3587 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
3588 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
3589 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
3590 NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
3591 NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
3592 NEONMAP1(vclz_v, ctlz, Add1ArgType),
3593 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
3594 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
3595 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
3596 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
3597 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
3598 NEONMAP0(vcvt_f32_v),
3599 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3600 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3601 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
3602 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
3603 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
3604 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
3605 NEONMAP0(vcvtq_f32_v),
3606 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3607 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3608 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
3609 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
3610 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
3611 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
3612 NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType),
3613 NEONMAP0(vext_v),
3614 NEONMAP0(vextq_v),
3615 NEONMAP0(vfma_v),
3616 NEONMAP0(vfmaq_v),
3617 NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
3618 NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
3619 NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
3620 NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
3621 NEONMAP0(vmovl_v),
3622 NEONMAP0(vmovn_v),
3623 NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType),
3624 NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType),
3625 NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType),
3626 NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
3627 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
3628 NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType),
3629 NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType),
3630 NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType),
3631 NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
3632 NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
3633 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
3634 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
3635 NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType),
3636 NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType),
3637 NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType),
3638 NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts),
3639 NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType),
3640 NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType),
3641 NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType),
3642 NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType),
3643 NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType),
3644 NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
3645 NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
3646 NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
3647 NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
3648 NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts),
3649 NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
3650 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
3651 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
3652 NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
3653 NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
3654 NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType),
3655 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
3656 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
3657 NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType),
3658 NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType),
3659 NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
3660 NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
3661 NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
3662 NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
3663 NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
3664 NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
3665 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
3666 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
3667 NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType),
3668 NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType),
3669 NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType),
3670 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
3671 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
3672 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
3673 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
3674 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
3675 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
3676 NEONMAP0(vshl_n_v),
3677 NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
3678 NEONMAP0(vshll_n_v),
3679 NEONMAP0(vshlq_n_v),
3680 NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
3681 NEONMAP0(vshr_n_v),
3682 NEONMAP0(vshrn_n_v),
3683 NEONMAP0(vshrq_n_v),
3684 NEONMAP0(vsubhn_v),
3685 NEONMAP0(vtst_v),
3686 NEONMAP0(vtstq_v),
3687};
3688
3689static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = {
3690 NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType),
3691 NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType),
3692 NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType),
3693 NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
3694 NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
3695 NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
3696 NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
3697 NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
3698 NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
3699 NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3700 NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
3701 NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType),
3702 NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
3703 NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType),
3704 NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3705 NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3706 NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
3707 NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
3708 NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
3709 NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
3710 NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
3711 NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
3712 NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
3713 NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
3714 NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
3715 NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
3716 NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
3717 NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
3718 NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
3719 NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
3720 NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
3721 NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
3722 NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
3723 NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
3724 NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
3725 NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
3726 NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
3727 NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
3728 NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
3729 NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
3730 NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
3731 NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
3732 NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
3733 NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
3734 NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
3735 NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
3736 NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
3737 NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
3738 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
3739 NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3740 NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3741 NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3742 NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3743 NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
3744 NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
3745 NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3746 NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3747 NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
3748 NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
3749 NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3750 NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3751 NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3752 NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
3753 NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
3754 NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
3755 NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
3756 NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
3757 NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
3758 NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
3759 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
3760 NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType),
3761 NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType),
3762 NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3763 NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3764 NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3765 NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3766 NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3767 NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3768 NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3769 NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3770 NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
3771 NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
3772 NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
3773 NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType),
3774 NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
3775 NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType),
3776 NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
3777 NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
3778 NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType),
3779 NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType),
3780 NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
3781 NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
3782 NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType),
3783 NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType),
3784 NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors),
3785 NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType),
3786 NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors),
3787 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
3788 NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType),
3789 NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType),
3790 NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
3791 NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
3792 NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
3793 NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
3794 NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType),
3795 NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
3796 NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
3797 NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
3798 NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType),
3799 NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
3800 NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType),
3801 NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors),
3802 NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType),
3803 NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
3804 NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
3805 NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType),
3806 NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType),
3807 NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
3808 NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
3809 NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType),
3810 NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType),
3811 NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType),
3812 NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType),
3813 NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
3814 NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
3815 NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
3816 NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
3817 NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType),
3818 NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
3819 NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
3820 NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
3821 NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
3822 NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
3823 NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
3824 NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType),
3825 NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType),
3826 NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
3827 NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
3828 NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
3829 NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
3830 NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType),
3831 NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType),
3832 NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType),
3833 NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType),
3834 NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
3835 NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
3836 NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType),
3837 NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType),
3838 NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType),
3839 NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
3840 NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
3841 NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
3842 NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
3843 NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType),
3844 NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
3845 NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
3846 NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
3847 NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
3848 NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType),
3849 NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType),
3850 NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
3851 NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
3852 NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType),
3853 NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType),
3854 NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType),
3855 NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType),
3856 NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType),
3857 NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType),
3858 NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType),
3859 NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType),
3860 NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType),
3861 NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType),
3862 NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType),
3863 NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType),
3864 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
3865 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
3866 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
3867 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
3868 NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType),
3869 NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType),
3870 NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType),
3871 NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType),
3872 NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
3873 NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType),
3874 NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
3875 NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType),
3876 NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType),
3877 NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType),
3878 NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
3879 NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType),
3880 NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
3881 NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType),
3882};
3883
3884#undef NEONMAP0
3885#undef NEONMAP1
3886#undef NEONMAP2
3887
3888static bool NEONSIMDIntrinsicsProvenSorted = false;
3889
3890static bool AArch64SIMDIntrinsicsProvenSorted = false;
3891static bool AArch64SISDIntrinsicsProvenSorted = false;
3892
3893
3894static const NeonIntrinsicInfo *
3895findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap,
3896 unsigned BuiltinID, bool &MapProvenSorted) {
3897
3898#ifndef NDEBUG
3899 if (!MapProvenSorted) {
3900 assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap)))(static_cast <bool> (std::is_sorted(std::begin(IntrinsicMap
), std::end(IntrinsicMap))) ? void (0) : __assert_fail ("std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap))"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3900, __extension__ __PRETTY_FUNCTION__))
;
3901 MapProvenSorted = true;
3902 }
3903#endif
3904
3905 const NeonIntrinsicInfo *Builtin =
3906 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID);
3907
3908 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
3909 return Builtin;
3910
3911 return nullptr;
3912}
3913
3914Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
3915 unsigned Modifier,
3916 llvm::Type *ArgType,
3917 const CallExpr *E) {
3918 int VectorSize = 0;
3919 if (Modifier & Use64BitVectors)
3920 VectorSize = 64;
3921 else if (Modifier & Use128BitVectors)
3922 VectorSize = 128;
3923
3924 // Return type.
3925 SmallVector<llvm::Type *, 3> Tys;
3926 if (Modifier & AddRetType) {
3927 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
3928 if (Modifier & VectorizeRetType)
3929 Ty = llvm::VectorType::get(
3930 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
3931
3932 Tys.push_back(Ty);
3933 }
3934
3935 // Arguments.
3936 if (Modifier & VectorizeArgTypes) {
3937 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
3938 ArgType = llvm::VectorType::get(ArgType, Elts);
3939 }
3940
3941 if (Modifier & (Add1ArgType | Add2ArgTypes))
3942 Tys.push_back(ArgType);
3943
3944 if (Modifier & Add2ArgTypes)
3945 Tys.push_back(ArgType);
3946
3947 if (Modifier & InventFloatType)
3948 Tys.push_back(FloatTy);
3949
3950 return CGM.getIntrinsic(IntrinsicID, Tys);
3951}
3952
3953static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF,
3954 const NeonIntrinsicInfo &SISDInfo,
3955 SmallVectorImpl<Value *> &Ops,
3956 const CallExpr *E) {
3957 unsigned BuiltinID = SISDInfo.BuiltinID;
3958 unsigned int Int = SISDInfo.LLVMIntrinsic;
3959 unsigned Modifier = SISDInfo.TypeModifier;
3960 const char *s = SISDInfo.NameHint;
3961
3962 switch (BuiltinID) {
3963 case NEON::BI__builtin_neon_vcled_s64:
3964 case NEON::BI__builtin_neon_vcled_u64:
3965 case NEON::BI__builtin_neon_vcles_f32:
3966 case NEON::BI__builtin_neon_vcled_f64:
3967 case NEON::BI__builtin_neon_vcltd_s64:
3968 case NEON::BI__builtin_neon_vcltd_u64:
3969 case NEON::BI__builtin_neon_vclts_f32:
3970 case NEON::BI__builtin_neon_vcltd_f64:
3971 case NEON::BI__builtin_neon_vcales_f32:
3972 case NEON::BI__builtin_neon_vcaled_f64:
3973 case NEON::BI__builtin_neon_vcalts_f32:
3974 case NEON::BI__builtin_neon_vcaltd_f64:
3975 // Only one direction of comparisons actually exist, cmle is actually a cmge
3976 // with swapped operands. The table gives us the right intrinsic but we
3977 // still need to do the swap.
3978 std::swap(Ops[0], Ops[1]);
3979 break;
3980 }
3981
3982 assert(Int && "Generic code assumes a valid intrinsic")(static_cast <bool> (Int && "Generic code assumes a valid intrinsic"
) ? void (0) : __assert_fail ("Int && \"Generic code assumes a valid intrinsic\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3982, __extension__ __PRETTY_FUNCTION__))
;
3983
3984 // Determine the type(s) of this overloaded AArch64 intrinsic.
3985 const Expr *Arg = E->getArg(0);
3986 llvm::Type *ArgTy = CGF.ConvertType(Arg->getType());
3987 Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E);
3988
3989 int j = 0;
3990 ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0);
3991 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
3992 ai != ae; ++ai, ++j) {
3993 llvm::Type *ArgTy = ai->getType();
3994 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
3995 ArgTy->getPrimitiveSizeInBits())
3996 continue;
3997
3998 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy())(static_cast <bool> (ArgTy->isVectorTy() && !
Ops[j]->getType()->isVectorTy()) ? void (0) : __assert_fail
("ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 3998, __extension__ __PRETTY_FUNCTION__))
;
3999 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
4000 // it before inserting.
4001 Ops[j] =
4002 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
4003 Ops[j] =
4004 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
4005 }
4006
4007 Value *Result = CGF.EmitNeonCall(F, Ops, s);
4008 llvm::Type *ResultType = CGF.ConvertType(E->getType());
4009 if (ResultType->getPrimitiveSizeInBits() <
4010 Result->getType()->getPrimitiveSizeInBits())
4011 return CGF.Builder.CreateExtractElement(Result, C0);
4012
4013 return CGF.Builder.CreateBitCast(Result, ResultType, s);
4014}
4015
4016Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
4017 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
4018 const char *NameHint, unsigned Modifier, const CallExpr *E,
4019 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) {
4020 // Get the last argument, which specifies the vector type.
4021 llvm::APSInt NeonTypeConst;
4022 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
4023 if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext()))
4024 return nullptr;
4025
4026 // Determine the type of this overloaded NEON intrinsic.
4027 NeonTypeFlags Type(NeonTypeConst.getZExtValue());
4028 bool Usgn = Type.isUnsigned();
4029 bool Quad = Type.isQuad();
4030
4031 llvm::VectorType *VTy = GetNeonType(this, Type);
4032 llvm::Type *Ty = VTy;
4033 if (!Ty)
4034 return nullptr;
4035
4036 auto getAlignmentValue32 = [&](Address addr) -> Value* {
4037 return Builder.getInt32(addr.getAlignment().getQuantity());
4038 };
4039
4040 unsigned Int = LLVMIntrinsic;
4041 if ((Modifier & UnsignedAlts) && !Usgn)
4042 Int = AltLLVMIntrinsic;
4043
4044 switch (BuiltinID) {
4045 default: break;
4046 case NEON::BI__builtin_neon_vabs_v:
4047 case NEON::BI__builtin_neon_vabsq_v:
4048 if (VTy->getElementType()->isFloatingPointTy())
4049 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
4050 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
4051 case NEON::BI__builtin_neon_vaddhn_v: {
4052 llvm::VectorType *SrcTy =
4053 llvm::VectorType::getExtendedElementVectorType(VTy);
4054
4055 // %sum = add <4 x i32> %lhs, %rhs
4056 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4057 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
4058 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
4059
4060 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
4061 Constant *ShiftAmt =
4062 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
4063 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
4064
4065 // %res = trunc <4 x i32> %high to <4 x i16>
4066 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
4067 }
4068 case NEON::BI__builtin_neon_vcale_v:
4069 case NEON::BI__builtin_neon_vcaleq_v:
4070 case NEON::BI__builtin_neon_vcalt_v:
4071 case NEON::BI__builtin_neon_vcaltq_v:
4072 std::swap(Ops[0], Ops[1]);
4073 LLVM_FALLTHROUGH[[clang::fallthrough]];
4074 case NEON::BI__builtin_neon_vcage_v:
4075 case NEON::BI__builtin_neon_vcageq_v:
4076 case NEON::BI__builtin_neon_vcagt_v:
4077 case NEON::BI__builtin_neon_vcagtq_v: {
4078 llvm::Type *VecFlt = llvm::VectorType::get(
4079 VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy,
4080 VTy->getNumElements());
4081 llvm::Type *Tys[] = { VTy, VecFlt };
4082 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4083 return EmitNeonCall(F, Ops, NameHint);
4084 }
4085 case NEON::BI__builtin_neon_vclz_v:
4086 case NEON::BI__builtin_neon_vclzq_v:
4087 // We generate target-independent intrinsic, which needs a second argument
4088 // for whether or not clz of zero is undefined; on ARM it isn't.
4089 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
4090 break;
4091 case NEON::BI__builtin_neon_vcvt_f32_v:
4092 case NEON::BI__builtin_neon_vcvtq_f32_v:
4093 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4094 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad));
4095 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
4096 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
4097 case NEON::BI__builtin_neon_vcvt_n_f32_v:
4098 case NEON::BI__builtin_neon_vcvt_n_f64_v:
4099 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
4100 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
4101 llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
4102 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
4103 Function *F = CGM.getIntrinsic(Int, Tys);
4104 return EmitNeonCall(F, Ops, "vcvt_n");
4105 }
4106 case NEON::BI__builtin_neon_vcvt_n_s32_v:
4107 case NEON::BI__builtin_neon_vcvt_n_u32_v:
4108 case NEON::BI__builtin_neon_vcvt_n_s64_v:
4109 case NEON::BI__builtin_neon_vcvt_n_u64_v:
4110 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
4111 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
4112 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
4113 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
4114 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
4115 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4116 return EmitNeonCall(F, Ops, "vcvt_n");
4117 }
4118 case NEON::BI__builtin_neon_vcvt_s32_v:
4119 case NEON::BI__builtin_neon_vcvt_u32_v:
4120 case NEON::BI__builtin_neon_vcvt_s64_v:
4121 case NEON::BI__builtin_neon_vcvt_u64_v:
4122 case NEON::BI__builtin_neon_vcvtq_s32_v:
4123 case NEON::BI__builtin_neon_vcvtq_u32_v:
4124 case NEON::BI__builtin_neon_vcvtq_s64_v:
4125 case NEON::BI__builtin_neon_vcvtq_u64_v: {
4126 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
4127 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
4128 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
4129 }
4130 case NEON::BI__builtin_neon_vcvta_s32_v:
4131 case NEON::BI__builtin_neon_vcvta_s64_v:
4132 case NEON::BI__builtin_neon_vcvta_u32_v:
4133 case NEON::BI__builtin_neon_vcvta_u64_v:
4134 case NEON::BI__builtin_neon_vcvtaq_s32_v:
4135 case NEON::BI__builtin_neon_vcvtaq_s64_v:
4136 case NEON::BI__builtin_neon_vcvtaq_u32_v:
4137 case NEON::BI__builtin_neon_vcvtaq_u64_v:
4138 case NEON::BI__builtin_neon_vcvtn_s32_v:
4139 case NEON::BI__builtin_neon_vcvtn_s64_v:
4140 case NEON::BI__builtin_neon_vcvtn_u32_v:
4141 case NEON::BI__builtin_neon_vcvtn_u64_v:
4142 case NEON::BI__builtin_neon_vcvtnq_s32_v:
4143 case NEON::BI__builtin_neon_vcvtnq_s64_v:
4144 case NEON::BI__builtin_neon_vcvtnq_u32_v:
4145 case NEON::BI__builtin_neon_vcvtnq_u64_v:
4146 case NEON::BI__builtin_neon_vcvtp_s32_v:
4147 case NEON::BI__builtin_neon_vcvtp_s64_v:
4148 case NEON::BI__builtin_neon_vcvtp_u32_v:
4149 case NEON::BI__builtin_neon_vcvtp_u64_v:
4150 case NEON::BI__builtin_neon_vcvtpq_s32_v:
4151 case NEON::BI__builtin_neon_vcvtpq_s64_v:
4152 case NEON::BI__builtin_neon_vcvtpq_u32_v:
4153 case NEON::BI__builtin_neon_vcvtpq_u64_v:
4154 case NEON::BI__builtin_neon_vcvtm_s32_v:
4155 case NEON::BI__builtin_neon_vcvtm_s64_v:
4156 case NEON::BI__builtin_neon_vcvtm_u32_v:
4157 case NEON::BI__builtin_neon_vcvtm_u64_v:
4158 case NEON::BI__builtin_neon_vcvtmq_s32_v:
4159 case NEON::BI__builtin_neon_vcvtmq_s64_v:
4160 case NEON::BI__builtin_neon_vcvtmq_u32_v:
4161 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
4162 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
4163 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
4164 }
4165 case NEON::BI__builtin_neon_vext_v:
4166 case NEON::BI__builtin_neon_vextq_v: {
4167 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
4168 SmallVector<uint32_t, 16> Indices;
4169 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4170 Indices.push_back(i+CV);
4171
4172 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4173 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4174 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext");
4175 }
4176 case NEON::BI__builtin_neon_vfma_v:
4177 case NEON::BI__builtin_neon_vfmaq_v: {
4178 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
4179 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4180 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4181 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4182
4183 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
4184 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
4185 }
4186 case NEON::BI__builtin_neon_vld1_v:
4187 case NEON::BI__builtin_neon_vld1q_v: {
4188 llvm::Type *Tys[] = {Ty, Int8PtrTy};
4189 Ops.push_back(getAlignmentValue32(PtrOp0));
4190 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1");
4191 }
4192 case NEON::BI__builtin_neon_vld2_v:
4193 case NEON::BI__builtin_neon_vld2q_v:
4194 case NEON::BI__builtin_neon_vld3_v:
4195 case NEON::BI__builtin_neon_vld3q_v:
4196 case NEON::BI__builtin_neon_vld4_v:
4197 case NEON::BI__builtin_neon_vld4q_v: {
4198 llvm::Type *Tys[] = {Ty, Int8PtrTy};
4199 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4200 Value *Align = getAlignmentValue32(PtrOp1);
4201 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
4202 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4203 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4204 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
4205 }
4206 case NEON::BI__builtin_neon_vld1_dup_v:
4207 case NEON::BI__builtin_neon_vld1q_dup_v: {
4208 Value *V = UndefValue::get(Ty);
4209 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
4210 PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty);
4211 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
4212 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
4213 Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
4214 return EmitNeonSplat(Ops[0], CI);
4215 }
4216 case NEON::BI__builtin_neon_vld2_lane_v:
4217 case NEON::BI__builtin_neon_vld2q_lane_v:
4218 case NEON::BI__builtin_neon_vld3_lane_v:
4219 case NEON::BI__builtin_neon_vld3q_lane_v:
4220 case NEON::BI__builtin_neon_vld4_lane_v:
4221 case NEON::BI__builtin_neon_vld4q_lane_v: {
4222 llvm::Type *Tys[] = {Ty, Int8PtrTy};
4223 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4224 for (unsigned I = 2; I < Ops.size() - 1; ++I)
4225 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
4226 Ops.push_back(getAlignmentValue32(PtrOp1));
4227 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
4228 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4229 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4230 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
4231 }
4232 case NEON::BI__builtin_neon_vmovl_v: {
4233 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
4234 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
4235 if (Usgn)
4236 return Builder.CreateZExt(Ops[0], Ty, "vmovl");
4237 return Builder.CreateSExt(Ops[0], Ty, "vmovl");
4238 }
4239 case NEON::BI__builtin_neon_vmovn_v: {
4240 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
4241 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
4242 return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
4243 }
4244 case NEON::BI__builtin_neon_vmull_v:
4245 // FIXME: the integer vmull operations could be emitted in terms of pure
4246 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
4247 // hoisting the exts outside loops. Until global ISel comes along that can
4248 // see through such movement this leads to bad CodeGen. So we need an
4249 // intrinsic for now.
4250 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
4251 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
4252 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
4253 case NEON::BI__builtin_neon_vpadal_v:
4254 case NEON::BI__builtin_neon_vpadalq_v: {
4255 // The source operand type has twice as many elements of half the size.
4256 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
4257 llvm::Type *EltTy =
4258 llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
4259 llvm::Type *NarrowTy =
4260 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
4261 llvm::Type *Tys[2] = { Ty, NarrowTy };
4262 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
4263 }
4264 case NEON::BI__builtin_neon_vpaddl_v:
4265 case NEON::BI__builtin_neon_vpaddlq_v: {
4266 // The source operand type has twice as many elements of half the size.
4267 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
4268 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
4269 llvm::Type *NarrowTy =
4270 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
4271 llvm::Type *Tys[2] = { Ty, NarrowTy };
4272 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
4273 }
4274 case NEON::BI__builtin_neon_vqdmlal_v:
4275 case NEON::BI__builtin_neon_vqdmlsl_v: {
4276 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
4277 Ops[1] =
4278 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal");
4279 Ops.resize(2);
4280 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
4281 }
4282 case NEON::BI__builtin_neon_vqshl_n_v:
4283 case NEON::BI__builtin_neon_vqshlq_n_v:
4284 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
4285 1, false);
4286 case NEON::BI__builtin_neon_vqshlu_n_v:
4287 case NEON::BI__builtin_neon_vqshluq_n_v:
4288 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n",
4289 1, false);
4290 case NEON::BI__builtin_neon_vrecpe_v:
4291 case NEON::BI__builtin_neon_vrecpeq_v:
4292 case NEON::BI__builtin_neon_vrsqrte_v:
4293 case NEON::BI__builtin_neon_vrsqrteq_v:
4294 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
4295 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
4296
4297 case NEON::BI__builtin_neon_vrshr_n_v:
4298 case NEON::BI__builtin_neon_vrshrq_n_v:
4299 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n",
4300 1, true);
4301 case NEON::BI__builtin_neon_vshl_n_v:
4302 case NEON::BI__builtin_neon_vshlq_n_v:
4303 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
4304 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
4305 "vshl_n");
4306 case NEON::BI__builtin_neon_vshll_n_v: {
4307 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
4308 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4309 if (Usgn)
4310 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
4311 else
4312 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
4313 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
4314 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
4315 }
4316 case NEON::BI__builtin_neon_vshrn_n_v: {
4317 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
4318 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4319 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
4320 if (Usgn)
4321 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
4322 else
4323 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
4324 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
4325 }
4326 case NEON::BI__builtin_neon_vshr_n_v:
4327 case NEON::BI__builtin_neon_vshrq_n_v:
4328 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n");
4329 case NEON::BI__builtin_neon_vst1_v:
4330 case NEON::BI__builtin_neon_vst1q_v:
4331 case NEON::BI__builtin_neon_vst2_v:
4332 case NEON::BI__builtin_neon_vst2q_v:
4333 case NEON::BI__builtin_neon_vst3_v:
4334 case NEON::BI__builtin_neon_vst3q_v:
4335 case NEON::BI__builtin_neon_vst4_v:
4336 case NEON::BI__builtin_neon_vst4q_v:
4337 case NEON::BI__builtin_neon_vst2_lane_v:
4338 case NEON::BI__builtin_neon_vst2q_lane_v:
4339 case NEON::BI__builtin_neon_vst3_lane_v:
4340 case NEON::BI__builtin_neon_vst3q_lane_v:
4341 case NEON::BI__builtin_neon_vst4_lane_v:
4342 case NEON::BI__builtin_neon_vst4q_lane_v: {
4343 llvm::Type *Tys[] = {Int8PtrTy, Ty};
4344 Ops.push_back(getAlignmentValue32(PtrOp0));
4345 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
4346 }
4347 case NEON::BI__builtin_neon_vsubhn_v: {
4348 llvm::VectorType *SrcTy =
4349 llvm::VectorType::getExtendedElementVectorType(VTy);
4350
4351 // %sum = add <4 x i32> %lhs, %rhs
4352 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4353 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
4354 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
4355
4356 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
4357 Constant *ShiftAmt =
4358 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
4359 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
4360
4361 // %res = trunc <4 x i32> %high to <4 x i16>
4362 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
4363 }
4364 case NEON::BI__builtin_neon_vtrn_v:
4365 case NEON::BI__builtin_neon_vtrnq_v: {
4366 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4367 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4368 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4369 Value *SV = nullptr;
4370
4371 for (unsigned vi = 0; vi != 2; ++vi) {
4372 SmallVector<uint32_t, 16> Indices;
4373 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4374 Indices.push_back(i+vi);
4375 Indices.push_back(i+e+vi);
4376 }
4377 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4378 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
4379 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
4380 }
4381 return SV;
4382 }
4383 case NEON::BI__builtin_neon_vtst_v:
4384 case NEON::BI__builtin_neon_vtstq_v: {
4385 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4386 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4387 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
4388 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
4389 ConstantAggregateZero::get(Ty));
4390 return Builder.CreateSExt(Ops[0], Ty, "vtst");
4391 }
4392 case NEON::BI__builtin_neon_vuzp_v:
4393 case NEON::BI__builtin_neon_vuzpq_v: {
4394 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4395 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4396 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4397 Value *SV = nullptr;
4398
4399 for (unsigned vi = 0; vi != 2; ++vi) {
4400 SmallVector<uint32_t, 16> Indices;
4401 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4402 Indices.push_back(2*i+vi);
4403
4404 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4405 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
4406 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
4407 }
4408 return SV;
4409 }
4410 case NEON::BI__builtin_neon_vzip_v:
4411 case NEON::BI__builtin_neon_vzipq_v: {
4412 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4413 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4414 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4415 Value *SV = nullptr;
4416
4417 for (unsigned vi = 0; vi != 2; ++vi) {
4418 SmallVector<uint32_t, 16> Indices;
4419 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4420 Indices.push_back((i + vi*e) >> 1);
4421 Indices.push_back(((i + vi*e) >> 1)+e);
4422 }
4423 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4424 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
4425 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
4426 }
4427 return SV;
4428 }
4429 }
4430
4431 assert(Int && "Expected valid intrinsic number")(static_cast <bool> (Int && "Expected valid intrinsic number"
) ? void (0) : __assert_fail ("Int && \"Expected valid intrinsic number\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4431, __extension__ __PRETTY_FUNCTION__))
;
4432
4433 // Determine the type(s) of this overloaded AArch64 intrinsic.
4434 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
4435
4436 Value *Result = EmitNeonCall(F, Ops, NameHint);
4437 llvm::Type *ResultType = ConvertType(E->getType());
4438 // AArch64 intrinsic one-element vector type cast to
4439 // scalar type expected by the builtin
4440 return Builder.CreateBitCast(Result, ResultType, NameHint);
4441}
4442
4443Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr(
4444 Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp,
4445 const CmpInst::Predicate Ip, const Twine &Name) {
4446 llvm::Type *OTy = Op->getType();
4447
4448 // FIXME: this is utterly horrific. We should not be looking at previous
4449 // codegen context to find out what needs doing. Unfortunately TableGen
4450 // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32
4451 // (etc).
4452 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
4453 OTy = BI->getOperand(0)->getType();
4454
4455 Op = Builder.CreateBitCast(Op, OTy);
4456 if (OTy->getScalarType()->isFloatingPointTy()) {
4457 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
4458 } else {
4459 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
4460 }
4461 return Builder.CreateSExt(Op, Ty, Name);
4462}
4463
4464static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
4465 Value *ExtOp, Value *IndexOp,
4466 llvm::Type *ResTy, unsigned IntID,
4467 const char *Name) {
4468 SmallVector<Value *, 2> TblOps;
4469 if (ExtOp)
4470 TblOps.push_back(ExtOp);
4471
4472 // Build a vector containing sequential number like (0, 1, 2, ..., 15)
4473 SmallVector<uint32_t, 16> Indices;
4474 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
4475 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
4476 Indices.push_back(2*i);
4477 Indices.push_back(2*i+1);
4478 }
4479
4480 int PairPos = 0, End = Ops.size() - 1;
4481 while (PairPos < End) {
4482 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
4483 Ops[PairPos+1], Indices,
4484 Name));
4485 PairPos += 2;
4486 }
4487
4488 // If there's an odd number of 64-bit lookup table, fill the high 64-bit
4489 // of the 128-bit lookup table with zero.
4490 if (PairPos == End) {
4491 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
4492 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
4493 ZeroTbl, Indices, Name));
4494 }
4495
4496 Function *TblF;
4497 TblOps.push_back(IndexOp);
4498 TblF = CGF.CGM.getIntrinsic(IntID, ResTy);
4499
4500 return CGF.EmitNeonCall(TblF, TblOps, Name);
4501}
4502
4503Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
4504 unsigned Value;
4505 switch (BuiltinID) {
4506 default:
4507 return nullptr;
4508 case ARM::BI__builtin_arm_nop:
4509 Value = 0;
4510 break;
4511 case ARM::BI__builtin_arm_yield:
4512 case ARM::BI__yield:
4513 Value = 1;
4514 break;
4515 case ARM::BI__builtin_arm_wfe:
4516 case ARM::BI__wfe:
4517 Value = 2;
4518 break;
4519 case ARM::BI__builtin_arm_wfi:
4520 case ARM::BI__wfi:
4521 Value = 3;
4522 break;
4523 case ARM::BI__builtin_arm_sev:
4524 case ARM::BI__sev:
4525 Value = 4;
4526 break;
4527 case ARM::BI__builtin_arm_sevl:
4528 case ARM::BI__sevl:
4529 Value = 5;
4530 break;
4531 }
4532
4533 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
4534 llvm::ConstantInt::get(Int32Ty, Value));
4535}
4536
4537// Generates the IR for the read/write special register builtin,
4538// ValueType is the type of the value that is to be written or read,
4539// RegisterType is the type of the register being written to or read from.
4540static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF,
4541 const CallExpr *E,
4542 llvm::Type *RegisterType,
4543 llvm::Type *ValueType,
4544 bool IsRead,
4545 StringRef SysReg = "") {
4546 // write and register intrinsics only support 32 and 64 bit operations.
4547 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))(static_cast <bool> ((RegisterType->isIntegerTy(32) ||
RegisterType->isIntegerTy(64)) && "Unsupported size for register."
) ? void (0) : __assert_fail ("(RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64)) && \"Unsupported size for register.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4548, __extension__ __PRETTY_FUNCTION__))
4548 && "Unsupported size for register.")(static_cast <bool> ((RegisterType->isIntegerTy(32) ||
RegisterType->isIntegerTy(64)) && "Unsupported size for register."
) ? void (0) : __assert_fail ("(RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64)) && \"Unsupported size for register.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4548, __extension__ __PRETTY_FUNCTION__))
;
4549
4550 CodeGen::CGBuilderTy &Builder = CGF.Builder;
4551 CodeGen::CodeGenModule &CGM = CGF.CGM;
4552 LLVMContext &Context = CGM.getLLVMContext();
4553
4554 if (SysReg.empty()) {
4555 const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts();
4556 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
4557 }
4558
4559 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
4560 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4561 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4562
4563 llvm::Type *Types[] = { RegisterType };
4564
4565 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
4566 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))(static_cast <bool> (!(RegisterType->isIntegerTy(32)
&& ValueType->isIntegerTy(64)) && "Can't fit 64-bit value in 32-bit register"
) ? void (0) : __assert_fail ("!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64)) && \"Can't fit 64-bit value in 32-bit register\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4567, __extension__ __PRETTY_FUNCTION__))
4567 && "Can't fit 64-bit value in 32-bit register")(static_cast <bool> (!(RegisterType->isIntegerTy(32)
&& ValueType->isIntegerTy(64)) && "Can't fit 64-bit value in 32-bit register"
) ? void (0) : __assert_fail ("!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64)) && \"Can't fit 64-bit value in 32-bit register\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4567, __extension__ __PRETTY_FUNCTION__))
;
4568
4569 if (IsRead) {
4570 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
4571 llvm::Value *Call = Builder.CreateCall(F, Metadata);
4572
4573 if (MixedTypes)
4574 // Read into 64 bit register and then truncate result to 32 bit.
4575 return Builder.CreateTrunc(Call, ValueType);
4576
4577 if (ValueType->isPointerTy())
4578 // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
4579 return Builder.CreateIntToPtr(Call, ValueType);
4580
4581 return Call;
4582 }
4583
4584 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
4585 llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1));
4586 if (MixedTypes) {
4587 // Extend 32 bit write value to 64 bit to pass to write.
4588 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
4589 return Builder.CreateCall(F, { Metadata, ArgValue });
4590 }
4591
4592 if (ValueType->isPointerTy()) {
4593 // Have VoidPtrTy ArgValue but want to return an i32/i64.
4594 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
4595 return Builder.CreateCall(F, { Metadata, ArgValue });
4596 }
4597
4598 return Builder.CreateCall(F, { Metadata, ArgValue });
4599}
4600
4601/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
4602/// argument that specifies the vector type.
4603static bool HasExtraNeonArgument(unsigned BuiltinID) {
4604 switch (BuiltinID) {
4605 default: break;
4606 case NEON::BI__builtin_neon_vget_lane_i8:
4607 case NEON::BI__builtin_neon_vget_lane_i16:
4608 case NEON::BI__builtin_neon_vget_lane_i32:
4609 case NEON::BI__builtin_neon_vget_lane_i64:
4610 case NEON::BI__builtin_neon_vget_lane_f32:
4611 case NEON::BI__builtin_neon_vgetq_lane_i8:
4612 case NEON::BI__builtin_neon_vgetq_lane_i16:
4613 case NEON::BI__builtin_neon_vgetq_lane_i32:
4614 case NEON::BI__builtin_neon_vgetq_lane_i64:
4615 case NEON::BI__builtin_neon_vgetq_lane_f32:
4616 case NEON::BI__builtin_neon_vset_lane_i8:
4617 case NEON::BI__builtin_neon_vset_lane_i16:
4618 case NEON::BI__builtin_neon_vset_lane_i32:
4619 case NEON::BI__builtin_neon_vset_lane_i64:
4620 case NEON::BI__builtin_neon_vset_lane_f32:
4621 case NEON::BI__builtin_neon_vsetq_lane_i8:
4622 case NEON::BI__builtin_neon_vsetq_lane_i16:
4623 case NEON::BI__builtin_neon_vsetq_lane_i32:
4624 case NEON::BI__builtin_neon_vsetq_lane_i64:
4625 case NEON::BI__builtin_neon_vsetq_lane_f32:
4626 case NEON::BI__builtin_neon_vsha1h_u32:
4627 case NEON::BI__builtin_neon_vsha1cq_u32:
4628 case NEON::BI__builtin_neon_vsha1pq_u32:
4629 case NEON::BI__builtin_neon_vsha1mq_u32:
4630 case clang::ARM::BI_MoveToCoprocessor:
4631 case clang::ARM::BI_MoveToCoprocessor2:
4632 return false;
4633 }
4634 return true;
4635}
4636
4637Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
4638 const CallExpr *E) {
4639 if (auto Hint = GetValueForARMHint(BuiltinID))
4640 return Hint;
4641
4642 if (BuiltinID == ARM::BI__emit) {
4643 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
4644 llvm::FunctionType *FTy =
4645 llvm::FunctionType::get(VoidTy, /*Variadic=*/false);
4646
4647 APSInt Value;
4648 if (!E->getArg(0)->EvaluateAsInt(Value, CGM.getContext()))
4649 llvm_unreachable("Sema will ensure that the parameter is constant")::llvm::llvm_unreachable_internal("Sema will ensure that the parameter is constant"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4649)
;
4650
4651 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
4652
4653 llvm::InlineAsm *Emit =
4654 IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "",
4655 /*SideEffects=*/true)
4656 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
4657 /*SideEffects=*/true);
4658
4659 return Builder.CreateCall(Emit);
4660 }
4661
4662 if (BuiltinID == ARM::BI__builtin_arm_dbg) {
4663 Value *Option = EmitScalarExpr(E->getArg(0));
4664 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
4665 }
4666
4667 if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
4668 Value *Address = EmitScalarExpr(E->getArg(0));
4669 Value *RW = EmitScalarExpr(E->getArg(1));
4670 Value *IsData = EmitScalarExpr(E->getArg(2));
4671
4672 // Locality is not supported on ARM target
4673 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
4674
4675 Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
4676 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
4677 }
4678
4679 if (BuiltinID == ARM::BI__builtin_arm_rbit) {
4680 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4681 return Builder.CreateCall(
4682 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
4683 }
4684
4685 if (BuiltinID == ARM::BI__clear_cache) {
4686 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments")(static_cast <bool> (E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments") ? void (0) : __assert_fail
("E->getNumArgs() == 2 && \"__clear_cache takes 2 arguments\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4686, __extension__ __PRETTY_FUNCTION__))
;
4687 const FunctionDecl *FD = E->getDirectCallee();
4688 Value *Ops[2];
4689 for (unsigned i = 0; i < 2; i++)
4690 Ops[i] = EmitScalarExpr(E->getArg(i));
4691 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
4692 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
4693 StringRef Name = FD->getName();
4694 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
4695 }
4696
4697 if (BuiltinID == ARM::BI__builtin_arm_mcrr ||
4698 BuiltinID == ARM::BI__builtin_arm_mcrr2) {
4699 Function *F;
4700
4701 switch (BuiltinID) {
4702 default: llvm_unreachable("unexpected builtin")::llvm::llvm_unreachable_internal("unexpected builtin", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4702)
;
4703 case ARM::BI__builtin_arm_mcrr:
4704 F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
4705 break;
4706 case ARM::BI__builtin_arm_mcrr2:
4707 F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
4708 break;
4709 }
4710
4711 // MCRR{2} instruction has 5 operands but
4712 // the intrinsic has 4 because Rt and Rt2
4713 // are represented as a single unsigned 64
4714 // bit integer in the intrinsic definition
4715 // but internally it's represented as 2 32
4716 // bit integers.
4717
4718 Value *Coproc = EmitScalarExpr(E->getArg(0));
4719 Value *Opc1 = EmitScalarExpr(E->getArg(1));
4720 Value *RtAndRt2 = EmitScalarExpr(E->getArg(2));
4721 Value *CRm = EmitScalarExpr(E->getArg(3));
4722
4723 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
4724 Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
4725 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
4726 Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
4727
4728 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
4729 }
4730
4731 if (BuiltinID == ARM::BI__builtin_arm_mrrc ||
4732 BuiltinID == ARM::BI__builtin_arm_mrrc2) {
4733 Function *F;
4734
4735 switch (BuiltinID) {
4736 default: llvm_unreachable("unexpected builtin")::llvm::llvm_unreachable_internal("unexpected builtin", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4736)
;
4737 case ARM::BI__builtin_arm_mrrc:
4738 F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
4739 break;
4740 case ARM::BI__builtin_arm_mrrc2:
4741 F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
4742 break;
4743 }
4744
4745 Value *Coproc = EmitScalarExpr(E->getArg(0));
4746 Value *Opc1 = EmitScalarExpr(E->getArg(1));
4747 Value *CRm = EmitScalarExpr(E->getArg(2));
4748 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
4749
4750 // Returns an unsigned 64 bit integer, represented
4751 // as two 32 bit integers.
4752
4753 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
4754 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
4755 Rt = Builder.CreateZExt(Rt, Int64Ty);
4756 Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
4757
4758 Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
4759 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast, "shl", true);
4760 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
4761
4762 return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType()));
4763 }
4764
4765 if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
4766 ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
4767 BuiltinID == ARM::BI__builtin_arm_ldaex) &&
4768 getContext().getTypeSize(E->getType()) == 64) ||
4769 BuiltinID == ARM::BI__ldrexd) {
4770 Function *F;
4771
4772 switch (BuiltinID) {
4773 default: llvm_unreachable("unexpected builtin")::llvm::llvm_unreachable_internal("unexpected builtin", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4773)
;
4774 case ARM::BI__builtin_arm_ldaex:
4775 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
4776 break;
4777 case ARM::BI__builtin_arm_ldrexd:
4778 case ARM::BI__builtin_arm_ldrex:
4779 case ARM::BI__ldrexd:
4780 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
4781 break;
4782 }
4783
4784 Value *LdPtr = EmitScalarExpr(E->getArg(0));
4785 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
4786 "ldrexd");
4787
4788 Value *Val0 = Builder.CreateExtractValue(Val, 1);
4789 Value *Val1 = Builder.CreateExtractValue(Val, 0);
4790 Val0 = Builder.CreateZExt(Val0, Int64Ty);
4791 Val1 = Builder.CreateZExt(Val1, Int64Ty);
4792
4793 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
4794 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
4795 Val = Builder.CreateOr(Val, Val1);
4796 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
4797 }
4798
4799 if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
4800 BuiltinID == ARM::BI__builtin_arm_ldaex) {
4801 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
4802
4803 QualType Ty = E->getType();
4804 llvm::Type *RealResTy = ConvertType(Ty);
4805 llvm::Type *PtrTy = llvm::IntegerType::get(
4806 getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
4807 LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
4808
4809 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
4810 ? Intrinsic::arm_ldaex
4811 : Intrinsic::arm_ldrex,
4812 PtrTy);
4813 Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
4814
4815 if (RealResTy->isPointerTy())
4816 return Builder.CreateIntToPtr(Val, RealResTy);
4817 else {
4818 llvm::Type *IntResTy = llvm::IntegerType::get(
4819 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
4820 Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
4821 return Builder.CreateBitCast(Val, RealResTy);
4822 }
4823 }
4824
4825 if (BuiltinID == ARM::BI__builtin_arm_strexd ||
4826 ((BuiltinID == ARM::BI__builtin_arm_stlex ||
4827 BuiltinID == ARM::BI__builtin_arm_strex) &&
4828 getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
4829 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
4830 ? Intrinsic::arm_stlexd
4831 : Intrinsic::arm_strexd);
4832 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
4833
4834 Address Tmp = CreateMemTemp(E->getArg(0)->getType());
4835 Value *Val = EmitScalarExpr(E->getArg(0));
4836 Builder.CreateStore(Val, Tmp);
4837
4838 Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
4839 Val = Builder.CreateLoad(LdPtr);
4840
4841 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
4842 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
4843 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy);
4844 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
4845 }
4846
4847 if (BuiltinID == ARM::BI__builtin_arm_strex ||
4848 BuiltinID == ARM::BI__builtin_arm_stlex) {
4849 Value *StoreVal = EmitScalarExpr(E->getArg(0));
4850 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
4851
4852 QualType Ty = E->getArg(0)->getType();
4853 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
4854 getContext().getTypeSize(Ty));
4855 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
4856
4857 if (StoreVal->getType()->isPointerTy())
4858 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
4859 else {
4860 llvm::Type *IntTy = llvm::IntegerType::get(
4861 getLLVMContext(),
4862 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
4863 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
4864 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
4865 }
4866
4867 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
4868 ? Intrinsic::arm_stlex
4869 : Intrinsic::arm_strex,
4870 StoreAddr->getType());
4871 return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
4872 }
4873
4874 switch (BuiltinID) {
4875 case ARM::BI__iso_volatile_load8:
4876 case ARM::BI__iso_volatile_load16:
4877 case ARM::BI__iso_volatile_load32:
4878 case ARM::BI__iso_volatile_load64: {
4879 Value *Ptr = EmitScalarExpr(E->getArg(0));
4880 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
4881 CharUnits LoadSize = getContext().getTypeSizeInChars(ElTy);
4882 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
4883 LoadSize.getQuantity() * 8);
4884 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
4885 llvm::LoadInst *Load =
4886 Builder.CreateAlignedLoad(Ptr, LoadSize);
4887 Load->setVolatile(true);
4888 return Load;
4889 }
4890 case ARM::BI__iso_volatile_store8:
4891 case ARM::BI__iso_volatile_store16:
4892 case ARM::BI__iso_volatile_store32:
4893 case ARM::BI__iso_volatile_store64: {
4894 Value *Ptr = EmitScalarExpr(E->getArg(0));
4895 Value *Value = EmitScalarExpr(E->getArg(1));
4896 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
4897 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
4898 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
4899 StoreSize.getQuantity() * 8);
4900 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
4901 llvm::StoreInst *Store =
4902 Builder.CreateAlignedStore(Value, Ptr,
4903 StoreSize);
4904 Store->setVolatile(true);
4905 return Store;
4906 }
4907 }
4908
4909 if (BuiltinID == ARM::BI__builtin_arm_clrex) {
4910 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
4911 return Builder.CreateCall(F);
4912 }
4913
4914 // CRC32
4915 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
4916 switch (BuiltinID) {
4917 case ARM::BI__builtin_arm_crc32b:
4918 CRCIntrinsicID = Intrinsic::arm_crc32b; break;
4919 case ARM::BI__builtin_arm_crc32cb:
4920 CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
4921 case ARM::BI__builtin_arm_crc32h:
4922 CRCIntrinsicID = Intrinsic::arm_crc32h; break;
4923 case ARM::BI__builtin_arm_crc32ch:
4924 CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
4925 case ARM::BI__builtin_arm_crc32w:
4926 case ARM::BI__builtin_arm_crc32d:
4927 CRCIntrinsicID = Intrinsic::arm_crc32w; break;
4928 case ARM::BI__builtin_arm_crc32cw:
4929 case ARM::BI__builtin_arm_crc32cd:
4930 CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
4931 }
4932
4933 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4934 Value *Arg0 = EmitScalarExpr(E->getArg(0));
4935 Value *Arg1 = EmitScalarExpr(E->getArg(1));
4936
4937 // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w
4938 // intrinsics, hence we need different codegen for these cases.
4939 if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
4940 BuiltinID == ARM::BI__builtin_arm_crc32cd) {
4941 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
4942 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
4943 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
4944 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
4945
4946 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
4947 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
4948 return Builder.CreateCall(F, {Res, Arg1b});
4949 } else {
4950 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
4951
4952 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
4953 return Builder.CreateCall(F, {Arg0, Arg1});
4954 }
4955 }
4956
4957 if (BuiltinID == ARM::BI__builtin_arm_rsr ||
4958 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4959 BuiltinID == ARM::BI__builtin_arm_rsrp ||
4960 BuiltinID == ARM::BI__builtin_arm_wsr ||
4961 BuiltinID == ARM::BI__builtin_arm_wsr64 ||
4962 BuiltinID == ARM::BI__builtin_arm_wsrp) {
4963
4964 bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr ||
4965 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4966 BuiltinID == ARM::BI__builtin_arm_rsrp;
4967
4968 bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp ||
4969 BuiltinID == ARM::BI__builtin_arm_wsrp;
4970
4971 bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4972 BuiltinID == ARM::BI__builtin_arm_wsr64;
4973
4974 llvm::Type *ValueType;
4975 llvm::Type *RegisterType;
4976 if (IsPointerBuiltin) {
4977 ValueType = VoidPtrTy;
4978 RegisterType = Int32Ty;
4979 } else if (Is64Bit) {
4980 ValueType = RegisterType = Int64Ty;
4981 } else {
4982 ValueType = RegisterType = Int32Ty;
4983 }
4984
4985 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead);
4986 }
4987
4988 // Find out if any arguments are required to be integer constant
4989 // expressions.
4990 unsigned ICEArguments = 0;
4991 ASTContext::GetBuiltinTypeError Error;
4992 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
4993 assert(Error == ASTContext::GE_None && "Should not codegen an error")(static_cast <bool> (Error == ASTContext::GE_None &&
"Should not codegen an error") ? void (0) : __assert_fail ("Error == ASTContext::GE_None && \"Should not codegen an error\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 4993, __extension__ __PRETTY_FUNCTION__))
;
4994
4995 auto getAlignmentValue32 = [&](Address addr) -> Value* {
4996 return Builder.getInt32(addr.getAlignment().getQuantity());
4997 };
4998
4999 Address PtrOp0 = Address::invalid();
5000 Address PtrOp1 = Address::invalid();
5001 SmallVector<Value*, 4> Ops;
5002 bool HasExtraArg = HasExtraNeonArgument(BuiltinID);
5003 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
5004 for (unsigned i = 0, e = NumArgs; i != e; i++) {
5005 if (i == 0) {
5006 switch (BuiltinID) {
5007 case NEON::BI__builtin_neon_vld1_v:
5008 case NEON::BI__builtin_neon_vld1q_v:
5009 case NEON::BI__builtin_neon_vld1q_lane_v:
5010 case NEON::BI__builtin_neon_vld1_lane_v:
5011 case NEON::BI__builtin_neon_vld1_dup_v:
5012 case NEON::BI__builtin_neon_vld1q_dup_v:
5013 case NEON::BI__builtin_neon_vst1_v:
5014 case NEON::BI__builtin_neon_vst1q_v:
5015 case NEON::BI__builtin_neon_vst1q_lane_v:
5016 case NEON::BI__builtin_neon_vst1_lane_v:
5017 case NEON::BI__builtin_neon_vst2_v:
5018 case NEON::BI__builtin_neon_vst2q_v:
5019 case NEON::BI__builtin_neon_vst2_lane_v:
5020 case NEON::BI__builtin_neon_vst2q_lane_v:
5021 case NEON::BI__builtin_neon_vst3_v:
5022 case NEON::BI__builtin_neon_vst3q_v:
5023 case NEON::BI__builtin_neon_vst3_lane_v:
5024 case NEON::BI__builtin_neon_vst3q_lane_v:
5025 case NEON::BI__builtin_neon_vst4_v:
5026 case NEON::BI__builtin_neon_vst4q_v:
5027 case NEON::BI__builtin_neon_vst4_lane_v:
5028 case NEON::BI__builtin_neon_vst4q_lane_v:
5029 // Get the alignment for the argument in addition to the value;
5030 // we'll use it later.
5031 PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
5032 Ops.push_back(PtrOp0.getPointer());
5033 continue;
5034 }
5035 }
5036 if (i == 1) {
5037 switch (BuiltinID) {
5038 case NEON::BI__builtin_neon_vld2_v:
5039 case NEON::BI__builtin_neon_vld2q_v:
5040 case NEON::BI__builtin_neon_vld3_v:
5041 case NEON::BI__builtin_neon_vld3q_v:
5042 case NEON::BI__builtin_neon_vld4_v:
5043 case NEON::BI__builtin_neon_vld4q_v:
5044 case NEON::BI__builtin_neon_vld2_lane_v:
5045 case NEON::BI__builtin_neon_vld2q_lane_v:
5046 case NEON::BI__builtin_neon_vld3_lane_v:
5047 case NEON::BI__builtin_neon_vld3q_lane_v:
5048 case NEON::BI__builtin_neon_vld4_lane_v:
5049 case NEON::BI__builtin_neon_vld4q_lane_v:
5050 case NEON::BI__builtin_neon_vld2_dup_v:
5051 case NEON::BI__builtin_neon_vld3_dup_v:
5052 case NEON::BI__builtin_neon_vld4_dup_v:
5053 // Get the alignment for the argument in addition to the value;
5054 // we'll use it later.
5055 PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
5056 Ops.push_back(PtrOp1.getPointer());
5057 continue;
5058 }
5059 }
5060
5061 if ((ICEArguments & (1 << i)) == 0) {
5062 Ops.push_back(EmitScalarExpr(E->getArg(i)));
5063 } else {
5064 // If this is required to be a constant, constant fold it so that we know
5065 // that the generated intrinsic gets a ConstantInt.
5066 llvm::APSInt Result;
5067 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
5068 assert(IsConst && "Constant arg isn't actually constant?")(static_cast <bool> (IsConst && "Constant arg isn't actually constant?"
) ? void (0) : __assert_fail ("IsConst && \"Constant arg isn't actually constant?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5068, __extension__ __PRETTY_FUNCTION__))
; (void)IsConst;
5069 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
5070 }
5071 }
5072
5073 switch (BuiltinID) {
5074 default: break;
5075
5076 case NEON::BI__builtin_neon_vget_lane_i8:
5077 case NEON::BI__builtin_neon_vget_lane_i16:
5078 case NEON::BI__builtin_neon_vget_lane_i32:
5079 case NEON::BI__builtin_neon_vget_lane_i64:
5080 case NEON::BI__builtin_neon_vget_lane_f32:
5081 case NEON::BI__builtin_neon_vgetq_lane_i8:
5082 case NEON::BI__builtin_neon_vgetq_lane_i16:
5083 case NEON::BI__builtin_neon_vgetq_lane_i32:
5084 case NEON::BI__builtin_neon_vgetq_lane_i64:
5085 case NEON::BI__builtin_neon_vgetq_lane_f32:
5086 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5087
5088 case NEON::BI__builtin_neon_vset_lane_i8:
5089 case NEON::BI__builtin_neon_vset_lane_i16:
5090 case NEON::BI__builtin_neon_vset_lane_i32:
5091 case NEON::BI__builtin_neon_vset_lane_i64:
5092 case NEON::BI__builtin_neon_vset_lane_f32:
5093 case NEON::BI__builtin_neon_vsetq_lane_i8:
5094 case NEON::BI__builtin_neon_vsetq_lane_i16:
5095 case NEON::BI__builtin_neon_vsetq_lane_i32:
5096 case NEON::BI__builtin_neon_vsetq_lane_i64:
5097 case NEON::BI__builtin_neon_vsetq_lane_f32:
5098 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5099
5100 case NEON::BI__builtin_neon_vsha1h_u32:
5101 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
5102 "vsha1h");
5103 case NEON::BI__builtin_neon_vsha1cq_u32:
5104 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
5105 "vsha1h");
5106 case NEON::BI__builtin_neon_vsha1pq_u32:
5107 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
5108 "vsha1h");
5109 case NEON::BI__builtin_neon_vsha1mq_u32:
5110 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
5111 "vsha1h");
5112
5113 // The ARM _MoveToCoprocessor builtins put the input register value as
5114 // the first argument, but the LLVM intrinsic expects it as the third one.
5115 case ARM::BI_MoveToCoprocessor:
5116 case ARM::BI_MoveToCoprocessor2: {
5117 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
5118 Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
5119 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
5120 Ops[3], Ops[4], Ops[5]});
5121 }
5122 case ARM::BI_BitScanForward:
5123 case ARM::BI_BitScanForward64:
5124 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
5125 case ARM::BI_BitScanReverse:
5126 case ARM::BI_BitScanReverse64:
5127 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
5128
5129 case ARM::BI_InterlockedAnd64:
5130 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
5131 case ARM::BI_InterlockedExchange64:
5132 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
5133 case ARM::BI_InterlockedExchangeAdd64:
5134 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
5135 case ARM::BI_InterlockedExchangeSub64:
5136 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
5137 case ARM::BI_InterlockedOr64:
5138 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
5139 case ARM::BI_InterlockedXor64:
5140 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
5141 case ARM::BI_InterlockedDecrement64:
5142 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
5143 case ARM::BI_InterlockedIncrement64:
5144 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
5145 }
5146
5147 // Get the last argument, which specifies the vector type.
5148 assert(HasExtraArg)(static_cast <bool> (HasExtraArg) ? void (0) : __assert_fail
("HasExtraArg", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5148, __extension__ __PRETTY_FUNCTION__))
;
5149 llvm::APSInt Result;
5150 const Expr *Arg = E->getArg(E->getNumArgs()-1);
5151 if (!Arg->isIntegerConstantExpr(Result, getContext()))
5152 return nullptr;
5153
5154 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
5155 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
5156 // Determine the overloaded type of this builtin.
5157 llvm::Type *Ty;
5158 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
5159 Ty = FloatTy;
5160 else
5161 Ty = DoubleTy;
5162
5163 // Determine whether this is an unsigned conversion or not.
5164 bool usgn = Result.getZExtValue() == 1;
5165 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
5166
5167 // Call the appropriate intrinsic.
5168 Function *F = CGM.getIntrinsic(Int, Ty);
5169 return Builder.CreateCall(F, Ops, "vcvtr");
5170 }
5171
5172 // Determine the type of this overloaded NEON intrinsic.
5173 NeonTypeFlags Type(Result.getZExtValue());
5174 bool usgn = Type.isUnsigned();
5175 bool rightShift = false;
5176
5177 llvm::VectorType *VTy = GetNeonType(this, Type);
5178 llvm::Type *Ty = VTy;
5179 if (!Ty)
5180 return nullptr;
5181
5182 // Many NEON builtins have identical semantics and uses in ARM and
5183 // AArch64. Emit these in a single function.
5184 auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap);
5185 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap(
5186 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
5187 if (Builtin)
5188 return EmitCommonNeonBuiltinExpr(
5189 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
5190 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1);
5191
5192 unsigned Int;
5193 switch (BuiltinID) {
5194 default: return nullptr;
5195 case NEON::BI__builtin_neon_vld1q_lane_v:
5196 // Handle 64-bit integer elements as a special case. Use shuffles of
5197 // one-element vectors to avoid poor code for i64 in the backend.
5198 if (VTy->getElementType()->isIntegerTy(64)) {
5199 // Extract the other lane.
5200 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5201 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
5202 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
5203 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
5204 // Load the value as a one-element vector.
5205 Ty = llvm::VectorType::get(VTy->getElementType(), 1);
5206 llvm::Type *Tys[] = {Ty, Int8PtrTy};
5207 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
5208 Value *Align = getAlignmentValue32(PtrOp0);
5209 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
5210 // Combine them.
5211 uint32_t Indices[] = {1 - Lane, Lane};
5212 SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices);
5213 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane");
5214 }
5215 // fall through
5216 case NEON::BI__builtin_neon_vld1_lane_v: {
5217 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5218 PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType());
5219 Value *Ld = Builder.CreateLoad(PtrOp0);
5220 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
5221 }
5222 case NEON::BI__builtin_neon_vld2_dup_v:
5223 case NEON::BI__builtin_neon_vld3_dup_v:
5224 case NEON::BI__builtin_neon_vld4_dup_v: {
5225 // Handle 64-bit elements as a special-case. There is no "dup" needed.
5226 if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) {
5227 switch (BuiltinID) {
5228 case NEON::BI__builtin_neon_vld2_dup_v:
5229 Int = Intrinsic::arm_neon_vld2;
5230 break;
5231 case NEON::BI__builtin_neon_vld3_dup_v:
5232 Int = Intrinsic::arm_neon_vld3;
5233 break;
5234 case NEON::BI__builtin_neon_vld4_dup_v:
5235 Int = Intrinsic::arm_neon_vld4;
5236 break;
5237 default: llvm_unreachable("unknown vld_dup intrinsic?")::llvm::llvm_unreachable_internal("unknown vld_dup intrinsic?"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5237)
;
5238 }
5239 llvm::Type *Tys[] = {Ty, Int8PtrTy};
5240 Function *F = CGM.getIntrinsic(Int, Tys);
5241 llvm::Value *Align = getAlignmentValue32(PtrOp1);
5242 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup");
5243 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5244 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5245 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5246 }
5247 switch (BuiltinID) {
5248 case NEON::BI__builtin_neon_vld2_dup_v:
5249 Int = Intrinsic::arm_neon_vld2lane;
5250 break;
5251 case NEON::BI__builtin_neon_vld3_dup_v:
5252 Int = Intrinsic::arm_neon_vld3lane;
5253 break;
5254 case NEON::BI__builtin_neon_vld4_dup_v:
5255 Int = Intrinsic::arm_neon_vld4lane;
5256 break;
5257 default: llvm_unreachable("unknown vld_dup intrinsic?")::llvm::llvm_unreachable_internal("unknown vld_dup intrinsic?"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5257)
;
5258 }
5259 llvm::Type *Tys[] = {Ty, Int8PtrTy};
5260 Function *F = CGM.getIntrinsic(Int, Tys);
5261 llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
5262
5263 SmallVector<Value*, 6> Args;
5264 Args.push_back(Ops[1]);
5265 Args.append(STy->getNumElements(), UndefValue::get(Ty));
5266
5267 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
5268 Args.push_back(CI);
5269 Args.push_back(getAlignmentValue32(PtrOp1));
5270
5271 Ops[1] = Builder.CreateCall(F, Args, "vld_dup");
5272 // splat lane 0 to all elts in each vector of the result.
5273 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5274 Value *Val = Builder.CreateExtractValue(Ops[1], i);
5275 Value *Elt = Builder.CreateBitCast(Val, Ty);
5276 Elt = EmitNeonSplat(Elt, CI);
5277 Elt = Builder.CreateBitCast(Elt, Val->getType());
5278 Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i);
5279 }
5280 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5281 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5282 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5283 }
5284 case NEON::BI__builtin_neon_vqrshrn_n_v:
5285 Int =
5286 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
5287 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
5288 1, true);
5289 case NEON::BI__builtin_neon_vqrshrun_n_v:
5290 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
5291 Ops, "vqrshrun_n", 1, true);
5292 case NEON::BI__builtin_neon_vqshrn_n_v:
5293 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
5294 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
5295 1, true);
5296 case NEON::BI__builtin_neon_vqshrun_n_v:
5297 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
5298 Ops, "vqshrun_n", 1, true);
5299 case NEON::BI__builtin_neon_vrecpe_v:
5300 case NEON::BI__builtin_neon_vrecpeq_v:
5301 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
5302 Ops, "vrecpe");
5303 case NEON::BI__builtin_neon_vrshrn_n_v:
5304 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
5305 Ops, "vrshrn_n", 1, true);
5306 case NEON::BI__builtin_neon_vrsra_n_v:
5307 case NEON::BI__builtin_neon_vrsraq_n_v:
5308 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5309 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5310 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
5311 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
5312 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
5313 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
5314 case NEON::BI__builtin_neon_vsri_n_v:
5315 case NEON::BI__builtin_neon_vsriq_n_v:
5316 rightShift = true;
5317 LLVM_FALLTHROUGH[[clang::fallthrough]];
5318 case NEON::BI__builtin_neon_vsli_n_v:
5319 case NEON::BI__builtin_neon_vsliq_n_v:
5320 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
5321 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
5322 Ops, "vsli_n");
5323 case NEON::BI__builtin_neon_vsra_n_v:
5324 case NEON::BI__builtin_neon_vsraq_n_v:
5325 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5326 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
5327 return Builder.CreateAdd(Ops[0], Ops[1]);
5328 case NEON::BI__builtin_neon_vst1q_lane_v:
5329 // Handle 64-bit integer elements as a special case. Use a shuffle to get
5330 // a one-element vector and avoid poor code for i64 in the backend.
5331 if (VTy->getElementType()->isIntegerTy(64)) {
5332 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5333 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
5334 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
5335 Ops[2] = getAlignmentValue32(PtrOp0);
5336 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
5337 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
5338 Tys), Ops);
5339 }
5340 // fall through
5341 case NEON::BI__builtin_neon_vst1_lane_v: {
5342 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5343 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
5344 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5345 auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty));
5346 return St;
5347 }
5348 case NEON::BI__builtin_neon_vtbl1_v:
5349 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
5350 Ops, "vtbl1");
5351 case NEON::BI__builtin_neon_vtbl2_v:
5352 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
5353 Ops, "vtbl2");
5354 case NEON::BI__builtin_neon_vtbl3_v:
5355 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
5356 Ops, "vtbl3");
5357 case NEON::BI__builtin_neon_vtbl4_v:
5358 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
5359 Ops, "vtbl4");
5360 case NEON::BI__builtin_neon_vtbx1_v:
5361 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
5362 Ops, "vtbx1");
5363 case NEON::BI__builtin_neon_vtbx2_v:
5364 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
5365 Ops, "vtbx2");
5366 case NEON::BI__builtin_neon_vtbx3_v:
5367 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
5368 Ops, "vtbx3");
5369 case NEON::BI__builtin_neon_vtbx4_v:
5370 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
5371 Ops, "vtbx4");
5372 }
5373}
5374
5375static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
5376 const CallExpr *E,
5377 SmallVectorImpl<Value *> &Ops) {
5378 unsigned int Int = 0;
5379 const char *s = nullptr;
5380
5381 switch (BuiltinID) {
5382 default:
5383 return nullptr;
5384 case NEON::BI__builtin_neon_vtbl1_v:
5385 case NEON::BI__builtin_neon_vqtbl1_v:
5386 case NEON::BI__builtin_neon_vqtbl1q_v:
5387 case NEON::BI__builtin_neon_vtbl2_v:
5388 case NEON::BI__builtin_neon_vqtbl2_v:
5389 case NEON::BI__builtin_neon_vqtbl2q_v:
5390 case NEON::BI__builtin_neon_vtbl3_v:
5391 case NEON::BI__builtin_neon_vqtbl3_v:
5392 case NEON::BI__builtin_neon_vqtbl3q_v:
5393 case NEON::BI__builtin_neon_vtbl4_v:
5394 case NEON::BI__builtin_neon_vqtbl4_v:
5395 case NEON::BI__builtin_neon_vqtbl4q_v:
5396 break;
5397 case NEON::BI__builtin_neon_vtbx1_v:
5398 case NEON::BI__builtin_neon_vqtbx1_v:
5399 case NEON::BI__builtin_neon_vqtbx1q_v:
5400 case NEON::BI__builtin_neon_vtbx2_v:
5401 case NEON::BI__builtin_neon_vqtbx2_v:
5402 case NEON::BI__builtin_neon_vqtbx2q_v:
5403 case NEON::BI__builtin_neon_vtbx3_v:
5404 case NEON::BI__builtin_neon_vqtbx3_v:
5405 case NEON::BI__builtin_neon_vqtbx3q_v:
5406 case NEON::BI__builtin_neon_vtbx4_v:
5407 case NEON::BI__builtin_neon_vqtbx4_v:
5408 case NEON::BI__builtin_neon_vqtbx4q_v:
5409 break;
5410 }
5411
5412 assert(E->getNumArgs() >= 3)(static_cast <bool> (E->getNumArgs() >= 3) ? void
(0) : __assert_fail ("E->getNumArgs() >= 3", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5412, __extension__ __PRETTY_FUNCTION__))
;
5413
5414 // Get the last argument, which specifies the vector type.
5415 llvm::APSInt Result;
5416 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
5417 if (!Arg->isIntegerConstantExpr(Result, CGF.getContext()))
5418 return nullptr;
5419
5420 // Determine the type of this overloaded NEON intrinsic.
5421 NeonTypeFlags Type(Result.getZExtValue());
5422 llvm::VectorType *Ty = GetNeonType(&CGF, Type);
5423 if (!Ty)
5424 return nullptr;
5425
5426 CodeGen::CGBuilderTy &Builder = CGF.Builder;
5427
5428 // AArch64 scalar builtins are not overloaded, they do not have an extra
5429 // argument that specifies the vector type, need to handle each case.
5430 switch (BuiltinID) {
5431 case NEON::BI__builtin_neon_vtbl1_v: {
5432 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr,
5433 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
5434 "vtbl1");
5435 }
5436 case NEON::BI__builtin_neon_vtbl2_v: {
5437 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr,
5438 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
5439 "vtbl1");
5440 }
5441 case NEON::BI__builtin_neon_vtbl3_v: {
5442 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr,
5443 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
5444 "vtbl2");
5445 }
5446 case NEON::BI__builtin_neon_vtbl4_v: {
5447 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr,
5448 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
5449 "vtbl2");
5450 }
5451 case NEON::BI__builtin_neon_vtbx1_v: {
5452 Value *TblRes =
5453 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2],
5454 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
5455
5456 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
5457 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
5458 CmpRes = Builder.CreateSExt(CmpRes, Ty);
5459
5460 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
5461 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
5462 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
5463 }
5464 case NEON::BI__builtin_neon_vtbx2_v: {
5465 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0],
5466 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
5467 "vtbx1");
5468 }
5469 case NEON::BI__builtin_neon_vtbx3_v: {
5470 Value *TblRes =
5471 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4],
5472 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
5473
5474 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
5475 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
5476 TwentyFourV);
5477 CmpRes = Builder.CreateSExt(CmpRes, Ty);
5478
5479 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
5480 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
5481 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
5482 }
5483 case NEON::BI__builtin_neon_vtbx4_v: {
5484 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0],
5485 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
5486 "vtbx2");
5487 }
5488 case NEON::BI__builtin_neon_vqtbl1_v:
5489 case NEON::BI__builtin_neon_vqtbl1q_v:
5490 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
5491 case NEON::BI__builtin_neon_vqtbl2_v:
5492 case NEON::BI__builtin_neon_vqtbl2q_v: {
5493 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
5494 case NEON::BI__builtin_neon_vqtbl3_v:
5495 case NEON::BI__builtin_neon_vqtbl3q_v:
5496 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
5497 case NEON::BI__builtin_neon_vqtbl4_v:
5498 case NEON::BI__builtin_neon_vqtbl4q_v:
5499 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
5500 case NEON::BI__builtin_neon_vqtbx1_v:
5501 case NEON::BI__builtin_neon_vqtbx1q_v:
5502 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
5503 case NEON::BI__builtin_neon_vqtbx2_v:
5504 case NEON::BI__builtin_neon_vqtbx2q_v:
5505 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
5506 case NEON::BI__builtin_neon_vqtbx3_v:
5507 case NEON::BI__builtin_neon_vqtbx3q_v:
5508 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
5509 case NEON::BI__builtin_neon_vqtbx4_v:
5510 case NEON::BI__builtin_neon_vqtbx4q_v:
5511 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
5512 }
5513 }
5514
5515 if (!Int)
5516 return nullptr;
5517
5518 Function *F = CGF.CGM.getIntrinsic(Int, Ty);
5519 return CGF.EmitNeonCall(F, Ops, s);
5520}
5521
5522Value *CodeGenFunction::vectorWrapScalar16(Value *Op) {
5523 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4);
5524 Op = Builder.CreateBitCast(Op, Int16Ty);
5525 Value *V = UndefValue::get(VTy);
5526 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
5527 Op = Builder.CreateInsertElement(V, Op, CI);
5528 return Op;
5529}
5530
5531Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
5532 const CallExpr *E) {
5533 unsigned HintID = static_cast<unsigned>(-1);
5534 switch (BuiltinID) {
5535 default: break;
5536 case AArch64::BI__builtin_arm_nop:
5537 HintID = 0;
5538 break;
5539 case AArch64::BI__builtin_arm_yield:
5540 HintID = 1;
5541 break;
5542 case AArch64::BI__builtin_arm_wfe:
5543 HintID = 2;
5544 break;
5545 case AArch64::BI__builtin_arm_wfi:
5546 HintID = 3;
5547 break;
5548 case AArch64::BI__builtin_arm_sev:
5549 HintID = 4;
5550 break;
5551 case AArch64::BI__builtin_arm_sevl:
5552 HintID = 5;
5553 break;
5554 }
5555
5556 if (HintID != static_cast<unsigned>(-1)) {
5557 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
5558 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
5559 }
5560
5561 if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
5562 Value *Address = EmitScalarExpr(E->getArg(0));
5563 Value *RW = EmitScalarExpr(E->getArg(1));
5564 Value *CacheLevel = EmitScalarExpr(E->getArg(2));
5565 Value *RetentionPolicy = EmitScalarExpr(E->getArg(3));
5566 Value *IsData = EmitScalarExpr(E->getArg(4));
5567
5568 Value *Locality = nullptr;
5569 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
5570 // Temporal fetch, needs to convert cache level to locality.
5571 Locality = llvm::ConstantInt::get(Int32Ty,
5572 -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3);
5573 } else {
5574 // Streaming fetch.
5575 Locality = llvm::ConstantInt::get(Int32Ty, 0);
5576 }
5577
5578 // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify
5579 // PLDL3STRM or PLDL2STRM.
5580 Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
5581 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
5582 }
5583
5584 if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
5585 assert((getContext().getTypeSize(E->getType()) == 32) &&(static_cast <bool> ((getContext().getTypeSize(E->getType
()) == 32) && "rbit of unusual size!") ? void (0) : __assert_fail
("(getContext().getTypeSize(E->getType()) == 32) && \"rbit of unusual size!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5586, __extension__ __PRETTY_FUNCTION__))
5586 "rbit of unusual size!")(static_cast <bool> ((getContext().getTypeSize(E->getType
()) == 32) && "rbit of unusual size!") ? void (0) : __assert_fail
("(getContext().getTypeSize(E->getType()) == 32) && \"rbit of unusual size!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5586, __extension__ __PRETTY_FUNCTION__))
;
5587 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
5588 return Builder.CreateCall(
5589 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
5590 }
5591 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
5592 assert((getContext().getTypeSize(E->getType()) == 64) &&(static_cast <bool> ((getContext().getTypeSize(E->getType
()) == 64) && "rbit of unusual size!") ? void (0) : __assert_fail
("(getContext().getTypeSize(E->getType()) == 64) && \"rbit of unusual size!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5593, __extension__ __PRETTY_FUNCTION__))
5593 "rbit of unusual size!")(static_cast <bool> ((getContext().getTypeSize(E->getType
()) == 64) && "rbit of unusual size!") ? void (0) : __assert_fail
("(getContext().getTypeSize(E->getType()) == 64) && \"rbit of unusual size!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5593, __extension__ __PRETTY_FUNCTION__))
;
5594 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
5595 return Builder.CreateCall(
5596 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
5597 }
5598
5599 if (BuiltinID == AArch64::BI__clear_cache) {
5600 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments")(static_cast <bool> (E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments") ? void (0) : __assert_fail
("E->getNumArgs() == 2 && \"__clear_cache takes 2 arguments\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5600, __extension__ __PRETTY_FUNCTION__))
;
5601 const FunctionDecl *FD = E->getDirectCallee();
5602 Value *Ops[2];
5603 for (unsigned i = 0; i < 2; i++)
5604 Ops[i] = EmitScalarExpr(E->getArg(i));
5605 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
5606 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
5607 StringRef Name = FD->getName();
5608 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
5609 }
5610
5611 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
5612 BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
5613 getContext().getTypeSize(E->getType()) == 128) {
5614 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
5615 ? Intrinsic::aarch64_ldaxp
5616 : Intrinsic::aarch64_ldxp);
5617
5618 Value *LdPtr = EmitScalarExpr(E->getArg(0));
5619 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
5620 "ldxp");
5621
5622 Value *Val0 = Builder.CreateExtractValue(Val, 1);
5623 Value *Val1 = Builder.CreateExtractValue(Val, 0);
5624 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
5625 Val0 = Builder.CreateZExt(Val0, Int128Ty);
5626 Val1 = Builder.CreateZExt(Val1, Int128Ty);
5627
5628 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
5629 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
5630 Val = Builder.CreateOr(Val, Val1);
5631 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
5632 } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
5633 BuiltinID == AArch64::BI__builtin_arm_ldaex) {
5634 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
5635
5636 QualType Ty = E->getType();
5637 llvm::Type *RealResTy = ConvertType(Ty);
5638 llvm::Type *PtrTy = llvm::IntegerType::get(
5639 getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
5640 LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
5641
5642 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
5643 ? Intrinsic::aarch64_ldaxr
5644 : Intrinsic::aarch64_ldxr,
5645 PtrTy);
5646 Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
5647
5648 if (RealResTy->isPointerTy())
5649 return Builder.CreateIntToPtr(Val, RealResTy);
5650
5651 llvm::Type *IntResTy = llvm::IntegerType::get(
5652 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
5653 Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
5654 return Builder.CreateBitCast(Val, RealResTy);
5655 }
5656
5657 if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
5658 BuiltinID == AArch64::BI__builtin_arm_stlex) &&
5659 getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
5660 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
5661 ? Intrinsic::aarch64_stlxp
5662 : Intrinsic::aarch64_stxp);
5663 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
5664
5665 Address Tmp = CreateMemTemp(E->getArg(0)->getType());
5666 EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true);
5667
5668 Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy));
5669 llvm::Value *Val = Builder.CreateLoad(Tmp);
5670
5671 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
5672 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
5673 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)),
5674 Int8PtrTy);
5675 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
5676 }
5677
5678 if (BuiltinID == AArch64::BI__builtin_arm_strex ||
5679 BuiltinID == AArch64::BI__builtin_arm_stlex) {
5680 Value *StoreVal = EmitScalarExpr(E->getArg(0));
5681 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
5682
5683 QualType Ty = E->getArg(0)->getType();
5684 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
5685 getContext().getTypeSize(Ty));
5686 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
5687
5688 if (StoreVal->getType()->isPointerTy())
5689 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
5690 else {
5691 llvm::Type *IntTy = llvm::IntegerType::get(
5692 getLLVMContext(),
5693 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
5694 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
5695 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
5696 }
5697
5698 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
5699 ? Intrinsic::aarch64_stlxr
5700 : Intrinsic::aarch64_stxr,
5701 StoreAddr->getType());
5702 return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
5703 }
5704
5705 if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
5706 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
5707 return Builder.CreateCall(F);
5708 }
5709
5710 // CRC32
5711 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
5712 switch (BuiltinID) {
5713 case AArch64::BI__builtin_arm_crc32b:
5714 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
5715 case AArch64::BI__builtin_arm_crc32cb:
5716 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
5717 case AArch64::BI__builtin_arm_crc32h:
5718 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
5719 case AArch64::BI__builtin_arm_crc32ch:
5720 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
5721 case AArch64::BI__builtin_arm_crc32w:
5722 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
5723 case AArch64::BI__builtin_arm_crc32cw:
5724 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
5725 case AArch64::BI__builtin_arm_crc32d:
5726 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
5727 case AArch64::BI__builtin_arm_crc32cd:
5728 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
5729 }
5730
5731 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
5732 Value *Arg0 = EmitScalarExpr(E->getArg(0));
5733 Value *Arg1 = EmitScalarExpr(E->getArg(1));
5734 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
5735
5736 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
5737 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
5738
5739 return Builder.CreateCall(F, {Arg0, Arg1});
5740 }
5741
5742 if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
5743 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
5744 BuiltinID == AArch64::BI__builtin_arm_rsrp ||
5745 BuiltinID == AArch64::BI__builtin_arm_wsr ||
5746 BuiltinID == AArch64::BI__builtin_arm_wsr64 ||
5747 BuiltinID == AArch64::BI__builtin_arm_wsrp) {
5748
5749 bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr ||
5750 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
5751 BuiltinID == AArch64::BI__builtin_arm_rsrp;
5752
5753 bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp ||
5754 BuiltinID == AArch64::BI__builtin_arm_wsrp;
5755
5756 bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr &&
5757 BuiltinID != AArch64::BI__builtin_arm_wsr;
5758
5759 llvm::Type *ValueType;
5760 llvm::Type *RegisterType = Int64Ty;
5761 if (IsPointerBuiltin) {
5762 ValueType = VoidPtrTy;
5763 } else if (Is64Bit) {
5764 ValueType = Int64Ty;
5765 } else {
5766 ValueType = Int32Ty;
5767 }
5768
5769 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead);
5770 }
5771
5772 // Find out if any arguments are required to be integer constant
5773 // expressions.
5774 unsigned ICEArguments = 0;
5775 ASTContext::GetBuiltinTypeError Error;
5776 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
5777 assert(Error == ASTContext::GE_None && "Should not codegen an error")(static_cast <bool> (Error == ASTContext::GE_None &&
"Should not codegen an error") ? void (0) : __assert_fail ("Error == ASTContext::GE_None && \"Should not codegen an error\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5777, __extension__ __PRETTY_FUNCTION__))
;
5778
5779 llvm::SmallVector<Value*, 4> Ops;
5780 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
5781 if ((ICEArguments & (1 << i)) == 0) {
5782 Ops.push_back(EmitScalarExpr(E->getArg(i)));
5783 } else {
5784 // If this is required to be a constant, constant fold it so that we know
5785 // that the generated intrinsic gets a ConstantInt.
5786 llvm::APSInt Result;
5787 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
5788 assert(IsConst && "Constant arg isn't actually constant?")(static_cast <bool> (IsConst && "Constant arg isn't actually constant?"
) ? void (0) : __assert_fail ("IsConst && \"Constant arg isn't actually constant?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5788, __extension__ __PRETTY_FUNCTION__))
;
5789 (void)IsConst;
5790 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
5791 }
5792 }
5793
5794 auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap);
5795 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap(
5796 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
5797
5798 if (Builtin) {
5799 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1)));
5800 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E);
5801 assert(Result && "SISD intrinsic should have been handled")(static_cast <bool> (Result && "SISD intrinsic should have been handled"
) ? void (0) : __assert_fail ("Result && \"SISD intrinsic should have been handled\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5801, __extension__ __PRETTY_FUNCTION__))
;
5802 return Result;
5803 }
5804
5805 llvm::APSInt Result;
5806 const Expr *Arg = E->getArg(E->getNumArgs()-1);
5807 NeonTypeFlags Type(0);
5808 if (Arg->isIntegerConstantExpr(Result, getContext()))
5809 // Determine the type of this overloaded NEON intrinsic.
5810 Type = NeonTypeFlags(Result.getZExtValue());
5811
5812 bool usgn = Type.isUnsigned();
5813 bool quad = Type.isQuad();
5814
5815 // Handle non-overloaded intrinsics first.
5816 switch (BuiltinID) {
5817 default: break;
5818 case NEON::BI__builtin_neon_vldrq_p128: {
5819 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
5820 llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0);
5821 Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy);
5822 return Builder.CreateAlignedLoad(Int128Ty, Ptr,
5823 CharUnits::fromQuantity(16));
5824 }
5825 case NEON::BI__builtin_neon_vstrq_p128: {
5826 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
5827 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
5828 return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr);
5829 }
5830 case NEON::BI__builtin_neon_vcvts_u32_f32:
5831 case NEON::BI__builtin_neon_vcvtd_u64_f64:
5832 usgn = true;
5833 // FALL THROUGH
5834 case NEON::BI__builtin_neon_vcvts_s32_f32:
5835 case NEON::BI__builtin_neon_vcvtd_s64_f64: {
5836 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5837 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5838 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
5839 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
5840 Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
5841 if (usgn)
5842 return Builder.CreateFPToUI(Ops[0], InTy);
5843 return Builder.CreateFPToSI(Ops[0], InTy);
5844 }
5845 case NEON::BI__builtin_neon_vcvts_f32_u32:
5846 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5847 usgn = true;
5848 // FALL THROUGH
5849 case NEON::BI__builtin_neon_vcvts_f32_s32:
5850 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5851 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5852 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5853 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
5854 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
5855 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
5856 if (usgn)
5857 return Builder.CreateUIToFP(Ops[0], FTy);
5858 return Builder.CreateSIToFP(Ops[0], FTy);
5859 }
5860 case NEON::BI__builtin_neon_vpaddd_s64: {
5861 llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2);
5862 Value *Vec = EmitScalarExpr(E->getArg(0));
5863 // The vector is v2f64, so make sure it's bitcast to that.
5864 Vec = Builder.CreateBitCast(Vec, Ty, "v2i64");
5865 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5866 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5867 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
5868 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
5869 // Pairwise addition of a v2f64 into a scalar f64.
5870 return Builder.CreateAdd(Op0, Op1, "vpaddd");
5871 }
5872 case NEON::BI__builtin_neon_vpaddd_f64: {
5873 llvm::Type *Ty =
5874 llvm::VectorType::get(DoubleTy, 2);
5875 Value *Vec = EmitScalarExpr(E->getArg(0));
5876 // The vector is v2f64, so make sure it's bitcast to that.
5877 Vec = Builder.CreateBitCast(Vec, Ty, "v2f64");
5878 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5879 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5880 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
5881 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
5882 // Pairwise addition of a v2f64 into a scalar f64.
5883 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
5884 }
5885 case NEON::BI__builtin_neon_vpadds_f32: {
5886 llvm::Type *Ty =
5887 llvm::VectorType::get(FloatTy, 2);
5888 Value *Vec = EmitScalarExpr(E->getArg(0));
5889 // The vector is v2f32, so make sure it's bitcast to that.
5890 Vec = Builder.CreateBitCast(Vec, Ty, "v2f32");
5891 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5892 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5893 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
5894 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
5895 // Pairwise addition of a v2f32 into a scalar f32.
5896 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
5897 }
5898 case NEON::BI__builtin_neon_vceqzd_s64:
5899 case NEON::BI__builtin_neon_vceqzd_f64:
5900 case NEON::BI__builtin_neon_vceqzs_f32:
5901 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5902 return EmitAArch64CompareBuiltinExpr(
5903 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5904 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz");
5905 case NEON::BI__builtin_neon_vcgezd_s64:
5906 case NEON::BI__builtin_neon_vcgezd_f64:
5907 case NEON::BI__builtin_neon_vcgezs_f32:
5908 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5909 return EmitAArch64CompareBuiltinExpr(
5910 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5911 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez");
5912 case NEON::BI__builtin_neon_vclezd_s64:
5913 case NEON::BI__builtin_neon_vclezd_f64:
5914 case NEON::BI__builtin_neon_vclezs_f32:
5915 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5916 return EmitAArch64CompareBuiltinExpr(
5917 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5918 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez");
5919 case NEON::BI__builtin_neon_vcgtzd_s64:
5920 case NEON::BI__builtin_neon_vcgtzd_f64:
5921 case NEON::BI__builtin_neon_vcgtzs_f32:
5922 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5923 return EmitAArch64CompareBuiltinExpr(
5924 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5925 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz");
5926 case NEON::BI__builtin_neon_vcltzd_s64:
5927 case NEON::BI__builtin_neon_vcltzd_f64:
5928 case NEON::BI__builtin_neon_vcltzs_f32:
5929 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5930 return EmitAArch64CompareBuiltinExpr(
5931 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5932 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz");
5933
5934 case NEON::BI__builtin_neon_vceqzd_u64: {
5935 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5936 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
5937 Ops[0] =
5938 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
5939 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd");
5940 }
5941 case NEON::BI__builtin_neon_vceqd_f64:
5942 case NEON::BI__builtin_neon_vcled_f64:
5943 case NEON::BI__builtin_neon_vcltd_f64:
5944 case NEON::BI__builtin_neon_vcged_f64:
5945 case NEON::BI__builtin_neon_vcgtd_f64: {
5946 llvm::CmpInst::Predicate P;
5947 switch (BuiltinID) {
5948 default: llvm_unreachable("missing builtin ID in switch!")::llvm::llvm_unreachable_internal("missing builtin ID in switch!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5948)
;
5949 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
5950 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
5951 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
5952 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
5953 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
5954 }
5955 Ops.push_back(EmitScalarExpr(E->getArg(1)));
5956 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
5957 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
5958 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5959 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
5960 }
5961 case NEON::BI__builtin_neon_vceqs_f32:
5962 case NEON::BI__builtin_neon_vcles_f32:
5963 case NEON::BI__builtin_neon_vclts_f32:
5964 case NEON::BI__builtin_neon_vcges_f32:
5965 case NEON::BI__builtin_neon_vcgts_f32: {
5966 llvm::CmpInst::Predicate P;
5967 switch (BuiltinID) {
5968 default: llvm_unreachable("missing builtin ID in switch!")::llvm::llvm_unreachable_internal("missing builtin ID in switch!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5968)
;
5969 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
5970 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
5971 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
5972 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
5973 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
5974 }
5975 Ops.push_back(EmitScalarExpr(E->getArg(1)));
5976 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
5977 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
5978 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5979 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
5980 }
5981 case NEON::BI__builtin_neon_vceqd_s64:
5982 case NEON::BI__builtin_neon_vceqd_u64:
5983 case NEON::BI__builtin_neon_vcgtd_s64:
5984 case NEON::BI__builtin_neon_vcgtd_u64:
5985 case NEON::BI__builtin_neon_vcltd_s64:
5986 case NEON::BI__builtin_neon_vcltd_u64:
5987 case NEON::BI__builtin_neon_vcged_u64:
5988 case NEON::BI__builtin_neon_vcged_s64:
5989 case NEON::BI__builtin_neon_vcled_u64:
5990 case NEON::BI__builtin_neon_vcled_s64: {
5991 llvm::CmpInst::Predicate P;
5992 switch (BuiltinID) {
5993 default: llvm_unreachable("missing builtin ID in switch!")::llvm::llvm_unreachable_internal("missing builtin ID in switch!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 5993)
;
5994 case NEON::BI__builtin_neon_vceqd_s64:
5995 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
5996 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
5997 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
5998 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
5999 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
6000 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
6001 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
6002 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
6003 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
6004 }
6005 Ops.push_back(EmitScalarExpr(E->getArg(1)));
6006 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
6007 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
6008 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
6009 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
6010 }
6011 case NEON::BI__builtin_neon_vtstd_s64:
6012 case NEON::BI__builtin_neon_vtstd_u64: {
6013 Ops.push_back(EmitScalarExpr(E->getArg(1)));
6014 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
6015 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
6016 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
6017 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
6018 llvm::Constant::getNullValue(Int64Ty));
6019 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
6020 }
6021 case NEON::BI__builtin_neon_vset_lane_i8:
6022 case NEON::BI__builtin_neon_vset_lane_i16:
6023 case NEON::BI__builtin_neon_vset_lane_i32:
6024 case NEON::BI__builtin_neon_vset_lane_i64:
6025 case NEON::BI__builtin_neon_vset_lane_f32:
6026 case NEON::BI__builtin_neon_vsetq_lane_i8:
6027 case NEON::BI__builtin_neon_vsetq_lane_i16:
6028 case NEON::BI__builtin_neon_vsetq_lane_i32:
6029 case NEON::BI__builtin_neon_vsetq_lane_i64:
6030 case NEON::BI__builtin_neon_vsetq_lane_f32:
6031 Ops.push_back(EmitScalarExpr(E->getArg(2)));
6032 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6033 case NEON::BI__builtin_neon_vset_lane_f64:
6034 // The vector type needs a cast for the v1f64 variant.
6035 Ops[1] = Builder.CreateBitCast(Ops[1],
6036 llvm::VectorType::get(DoubleTy, 1));
6037 Ops.push_back(EmitScalarExpr(E->getArg(2)));
6038 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6039 case NEON::BI__builtin_neon_vsetq_lane_f64:
6040 // The vector type needs a cast for the v2f64 variant.
6041 Ops[1] = Builder.CreateBitCast(Ops[1],
6042 llvm::VectorType::get(DoubleTy, 2));
6043 Ops.push_back(EmitScalarExpr(E->getArg(2)));
6044 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6045
6046 case NEON::BI__builtin_neon_vget_lane_i8:
6047 case NEON::BI__builtin_neon_vdupb_lane_i8:
6048 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8));
6049 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6050 "vget_lane");
6051 case NEON::BI__builtin_neon_vgetq_lane_i8:
6052 case NEON::BI__builtin_neon_vdupb_laneq_i8:
6053 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16));
6054 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6055 "vgetq_lane");
6056 case NEON::BI__builtin_neon_vget_lane_i16:
6057 case NEON::BI__builtin_neon_vduph_lane_i16:
6058 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4));
6059 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6060 "vget_lane");
6061 case NEON::BI__builtin_neon_vgetq_lane_i16:
6062 case NEON::BI__builtin_neon_vduph_laneq_i16:
6063 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8));
6064 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6065 "vgetq_lane");
6066 case NEON::BI__builtin_neon_vget_lane_i32:
6067 case NEON::BI__builtin_neon_vdups_lane_i32:
6068 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2));
6069 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6070 "vget_lane");
6071 case NEON::BI__builtin_neon_vdups_lane_f32:
6072 Ops[0] = Builder.CreateBitCast(Ops[0],
6073 llvm::VectorType::get(FloatTy, 2));
6074 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6075 "vdups_lane");
6076 case NEON::BI__builtin_neon_vgetq_lane_i32:
6077 case NEON::BI__builtin_neon_vdups_laneq_i32:
6078 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
6079 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6080 "vgetq_lane");
6081 case NEON::BI__builtin_neon_vget_lane_i64:
6082 case NEON::BI__builtin_neon_vdupd_lane_i64:
6083 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1));
6084 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6085 "vget_lane");
6086 case NEON::BI__builtin_neon_vdupd_lane_f64:
6087 Ops[0] = Builder.CreateBitCast(Ops[0],
6088 llvm::VectorType::get(DoubleTy, 1));
6089 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6090 "vdupd_lane");
6091 case NEON::BI__builtin_neon_vgetq_lane_i64:
6092 case NEON::BI__builtin_neon_vdupd_laneq_i64:
6093 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
6094 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6095 "vgetq_lane");
6096 case NEON::BI__builtin_neon_vget_lane_f32:
6097 Ops[0] = Builder.CreateBitCast(Ops[0],
6098 llvm::VectorType::get(FloatTy, 2));
6099 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6100 "vget_lane");
6101 case NEON::BI__builtin_neon_vget_lane_f64:
6102 Ops[0] = Builder.CreateBitCast(Ops[0],
6103 llvm::VectorType::get(DoubleTy, 1));
6104 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6105 "vget_lane");
6106 case NEON::BI__builtin_neon_vgetq_lane_f32:
6107 case NEON::BI__builtin_neon_vdups_laneq_f32:
6108 Ops[0] = Builder.CreateBitCast(Ops[0],
6109 llvm::VectorType::get(FloatTy, 4));
6110 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6111 "vgetq_lane");
6112 case NEON::BI__builtin_neon_vgetq_lane_f64:
6113 case NEON::BI__builtin_neon_vdupd_laneq_f64:
6114 Ops[0] = Builder.CreateBitCast(Ops[0],
6115 llvm::VectorType::get(DoubleTy, 2));
6116 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
6117 "vgetq_lane");
6118 case NEON::BI__builtin_neon_vaddd_s64:
6119 case NEON::BI__builtin_neon_vaddd_u64:
6120 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd");
6121 case NEON::BI__builtin_neon_vsubd_s64:
6122 case NEON::BI__builtin_neon_vsubd_u64:
6123 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd");
6124 case NEON::BI__builtin_neon_vqdmlalh_s16:
6125 case NEON::BI__builtin_neon_vqdmlslh_s16: {
6126 SmallVector<Value *, 2> ProductOps;
6127 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
6128 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2))));
6129 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
6130 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
6131 ProductOps, "vqdmlXl");
6132 Constant *CI = ConstantInt::get(SizeTy, 0);
6133 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
6134
6135 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
6136 ? Intrinsic::aarch64_neon_sqadd
6137 : Intrinsic::aarch64_neon_sqsub;
6138 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl");
6139 }
6140 case NEON::BI__builtin_neon_vqshlud_n_s64: {
6141 Ops.push_back(EmitScalarExpr(E->getArg(1)));
6142 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
6143 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
6144 Ops, "vqshlu_n");
6145 }
6146 case NEON::BI__builtin_neon_vqshld_n_u64:
6147 case NEON::BI__builtin_neon_vqshld_n_s64: {
6148 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
6149 ? Intrinsic::aarch64_neon_uqshl
6150 : Intrinsic::aarch64_neon_sqshl;
6151 Ops.push_back(EmitScalarExpr(E->getArg(1)));
6152 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
6153 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n");
6154 }
6155 case NEON::BI__builtin_neon_vrshrd_n_u64:
6156 case NEON::BI__builtin_neon_vrshrd_n_s64: {
6157 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
6158 ? Intrinsic::aarch64_neon_urshl
6159 : Intrinsic::aarch64_neon_srshl;
6160 Ops.push_back(EmitScalarExpr(E->getArg(1)));
6161 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
6162 Ops[1] = ConstantInt::get(Int64Ty, -SV);
6163 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n");
6164 }
6165 case NEON::BI__builtin_neon_vrsrad_n_u64:
6166 case NEON::BI__builtin_neon_vrsrad_n_s64: {
6167 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
6168 ? Intrinsic::aarch64_neon_urshl
6169 : Intrinsic::aarch64_neon_srshl;
6170 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
6171 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
6172 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
6173 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
6174 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
6175 }
6176 case NEON::BI__builtin_neon_vshld_n_s64:
6177 case NEON::BI__builtin_neon_vshld_n_u64: {
6178 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
6179 return Builder.CreateShl(
6180 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
6181 }
6182 case NEON::BI__builtin_neon_vshrd_n_s64: {
6183 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
6184 return Builder.CreateAShr(
6185 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
6186 Amt->getZExtValue())),
6187 "shrd_n");
6188 }
6189 case NEON::BI__builtin_neon_vshrd_n_u64: {
6190 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
6191 uint64_t ShiftAmt = Amt->getZExtValue();
6192 // Right-shifting an unsigned value by its size yields 0.
6193 if (ShiftAmt == 64)
6194 return ConstantInt::get(Int64Ty, 0);
6195 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
6196 "shrd_n");
6197 }
6198 case NEON::BI__builtin_neon_vsrad_n_s64: {
6199 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
6200 Ops[1] = Builder.CreateAShr(
6201 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
6202 Amt->getZExtValue())),
6203 "shrd_n");
6204 return Builder.CreateAdd(Ops[0], Ops[1]);
6205 }
6206 case NEON::BI__builtin_neon_vsrad_n_u64: {
6207 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
6208 uint64_t ShiftAmt = Amt->getZExtValue();
6209 // Right-shifting an unsigned value by its size yields 0.
6210 // As Op + 0 = Op, return Ops[0] directly.
6211 if (ShiftAmt == 64)
6212 return Ops[0];
6213 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
6214 "shrd_n");
6215 return Builder.CreateAdd(Ops[0], Ops[1]);
6216 }
6217 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
6218 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
6219 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
6220 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
6221 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
6222 "lane");
6223 SmallVector<Value *, 2> ProductOps;
6224 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
6225 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
6226 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
6227 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
6228 ProductOps, "vqdmlXl");
6229 Constant *CI = ConstantInt::get(SizeTy, 0);
6230 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
6231 Ops.pop_back();
6232
6233 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
6234 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
6235 ? Intrinsic::aarch64_neon_sqadd
6236 : Intrinsic::aarch64_neon_sqsub;
6237 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl");
6238 }
6239 case NEON::BI__builtin_neon_vqdmlals_s32:
6240 case NEON::BI__builtin_neon_vqdmlsls_s32: {
6241 SmallVector<Value *, 2> ProductOps;
6242 ProductOps.push_back(Ops[1]);
6243 ProductOps.push_back(EmitScalarExpr(E->getArg(2)));
6244 Ops[1] =
6245 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
6246 ProductOps, "vqdmlXl");
6247
6248 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
6249 ? Intrinsic::aarch64_neon_sqadd
6250 : Intrinsic::aarch64_neon_sqsub;
6251 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl");
6252 }
6253 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
6254 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
6255 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
6256 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
6257 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
6258 "lane");
6259 SmallVector<Value *, 2> ProductOps;
6260 ProductOps.push_back(Ops[1]);
6261 ProductOps.push_back(Ops[2]);
6262 Ops[1] =
6263 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
6264 ProductOps, "vqdmlXl");
6265 Ops.pop_back();
6266
6267 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
6268 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
6269 ? Intrinsic::aarch64_neon_sqadd
6270 : Intrinsic::aarch64_neon_sqsub;
6271 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
6272 }
6273 }
6274
6275 llvm::VectorType *VTy = GetNeonType(this, Type);
6276 llvm::Type *Ty = VTy;
6277 if (!Ty)
6278 return nullptr;
6279
6280 // Not all intrinsics handled by the common case work for AArch64 yet, so only
6281 // defer to common code if it's been added to our special map.
6282 Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID,
6283 AArch64SIMDIntrinsicsProvenSorted);
6284
6285 if (Builtin)
6286 return EmitCommonNeonBuiltinExpr(
6287 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
6288 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
6289 /*never use addresses*/ Address::invalid(), Address::invalid());
6290
6291 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops))
6292 return V;
6293
6294 unsigned Int;
6295 switch (BuiltinID) {
6296 default: return nullptr;
6297 case NEON::BI__builtin_neon_vbsl_v:
6298 case NEON::BI__builtin_neon_vbslq_v: {
6299 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
6300 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
6301 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
6302 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
6303
6304 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
6305 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
6306 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
6307 return Builder.CreateBitCast(Ops[0], Ty);
6308 }
6309 case NEON::BI__builtin_neon_vfma_lane_v:
6310 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
6311 // The ARM builtins (and instructions) have the addend as the first
6312 // operand, but the 'fma' intrinsics have it last. Swap it around here.
6313 Value *Addend = Ops[0];
6314 Value *Multiplicand = Ops[1];
6315 Value *LaneSource = Ops[2];
6316 Ops[0] = Multiplicand;
6317 Ops[1] = LaneSource;
6318 Ops[2] = Addend;
6319
6320 // Now adjust things to handle the lane access.
6321 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ?
6322 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
6323 VTy;
6324 llvm::Constant *cst = cast<Constant>(Ops[3]);
6325 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
6326 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
6327 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
6328
6329 Ops.pop_back();
6330 Int = Intrinsic::fma;
6331 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
6332 }
6333 case NEON::BI__builtin_neon_vfma_laneq_v: {
6334 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6335 // v1f64 fma should be mapped to Neon scalar f64 fma
6336 if (VTy && VTy->getElementType() == DoubleTy) {
6337 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6338 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
6339 llvm::Type *VTy = GetNeonType(this,
6340 NeonTypeFlags(NeonTypeFlags::Float64, false, true));
6341 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
6342 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
6343 Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy);
6344 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
6345 return Builder.CreateBitCast(Result, Ty);
6346 }
6347 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
6348 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6349 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6350
6351 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
6352 VTy->getNumElements() * 2);
6353 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
6354 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
6355 cast<ConstantInt>(Ops[3]));
6356 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
6357
6358 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
6359 }
6360 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
6361 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
6362 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6363 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6364
6365 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6366 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
6367 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
6368 }
6369 case NEON::BI__builtin_neon_vfmas_lane_f32:
6370 case NEON::BI__builtin_neon_vfmas_laneq_f32:
6371 case NEON::BI__builtin_neon_vfmad_lane_f64:
6372 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6373 Ops.push_back(EmitScalarExpr(E->getArg(3)));
6374 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
6375 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
6376 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
6377 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
6378 }
6379 case NEON::BI__builtin_neon_vmull_v:
6380 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6381 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6382 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
6383 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
6384 case NEON::BI__builtin_neon_vmax_v:
6385 case NEON::BI__builtin_neon_vmaxq_v:
6386 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6387 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
6388 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
6389 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
6390 case NEON::BI__builtin_neon_vmin_v:
6391 case NEON::BI__builtin_neon_vminq_v:
6392 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6393 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
6394 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
6395 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
6396 case NEON::BI__builtin_neon_vabd_v:
6397 case NEON::BI__builtin_neon_vabdq_v:
6398 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6399 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6400 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
6401 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
6402 case NEON::BI__builtin_neon_vpadal_v:
6403 case NEON::BI__builtin_neon_vpadalq_v: {
6404 unsigned ArgElts = VTy->getNumElements();
6405 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
6406 unsigned BitWidth = EltTy->getBitWidth();
6407 llvm::Type *ArgTy = llvm::VectorType::get(
6408 llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts);
6409 llvm::Type* Tys[2] = { VTy, ArgTy };
6410 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6411 SmallVector<llvm::Value*, 1> TmpOps;
6412 TmpOps.push_back(Ops[1]);
6413 Function *F = CGM.getIntrinsic(Int, Tys);
6414 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal");
6415 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
6416 return Builder.CreateAdd(tmp, addend);
6417 }
6418 case NEON::BI__builtin_neon_vpmin_v:
6419 case NEON::BI__builtin_neon_vpminq_v:
6420 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6421 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6422 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
6423 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
6424 case NEON::BI__builtin_neon_vpmax_v:
6425 case NEON::BI__builtin_neon_vpmaxq_v:
6426 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6427 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6428 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
6429 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
6430 case NEON::BI__builtin_neon_vminnm_v:
6431 case NEON::BI__builtin_neon_vminnmq_v:
6432 Int = Intrinsic::aarch64_neon_fminnm;
6433 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
6434 case NEON::BI__builtin_neon_vmaxnm_v:
6435 case NEON::BI__builtin_neon_vmaxnmq_v:
6436 Int = Intrinsic::aarch64_neon_fmaxnm;
6437 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
6438 case NEON::BI__builtin_neon_vrecpss_f32: {
6439 Ops.push_back(EmitScalarExpr(E->getArg(1)));
6440 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
6441 Ops, "vrecps");
6442 }
6443 case NEON::BI__builtin_neon_vrecpsd_f64: {
6444 Ops.push_back(EmitScalarExpr(E->getArg(1)));
6445 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
6446 Ops, "vrecps");
6447 }
6448 case NEON::BI__builtin_neon_vqshrun_n_v:
6449 Int = Intrinsic::aarch64_neon_sqshrun;
6450 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
6451 case NEON::BI__builtin_neon_vqrshrun_n_v:
6452 Int = Intrinsic::aarch64_neon_sqrshrun;
6453 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
6454 case NEON::BI__builtin_neon_vqshrn_n_v:
6455 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6456 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
6457 case NEON::BI__builtin_neon_vrshrn_n_v:
6458 Int = Intrinsic::aarch64_neon_rshrn;
6459 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
6460 case NEON::BI__builtin_neon_vqrshrn_n_v:
6461 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6462 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
6463 case NEON::BI__builtin_neon_vrnda_v:
6464 case NEON::BI__builtin_neon_vrndaq_v: {
6465 Int = Intrinsic::round;
6466 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
6467 }
6468 case NEON::BI__builtin_neon_vrndi_v:
6469 case NEON::BI__builtin_neon_vrndiq_v: {
6470 Int = Intrinsic::nearbyint;
6471 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi");
6472 }
6473 case NEON::BI__builtin_neon_vrndm_v:
6474 case NEON::BI__builtin_neon_vrndmq_v: {
6475 Int = Intrinsic::floor;
6476 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
6477 }
6478 case NEON::BI__builtin_neon_vrndn_v:
6479 case NEON::BI__builtin_neon_vrndnq_v: {
6480 Int = Intrinsic::aarch64_neon_frintn;
6481 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
6482 }
6483 case NEON::BI__builtin_neon_vrndp_v:
6484 case NEON::BI__builtin_neon_vrndpq_v: {
6485 Int = Intrinsic::ceil;
6486 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
6487 }
6488 case NEON::BI__builtin_neon_vrndx_v:
6489 case NEON::BI__builtin_neon_vrndxq_v: {
6490 Int = Intrinsic::rint;
6491 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
6492 }
6493 case NEON::BI__builtin_neon_vrnd_v:
6494 case NEON::BI__builtin_neon_vrndq_v: {
6495 Int = Intrinsic::trunc;
6496 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
6497 }
6498 case NEON::BI__builtin_neon_vceqz_v:
6499 case NEON::BI__builtin_neon_vceqzq_v:
6500 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
6501 ICmpInst::ICMP_EQ, "vceqz");
6502 case NEON::BI__builtin_neon_vcgez_v:
6503 case NEON::BI__builtin_neon_vcgezq_v:
6504 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
6505 ICmpInst::ICMP_SGE, "vcgez");
6506 case NEON::BI__builtin_neon_vclez_v:
6507 case NEON::BI__builtin_neon_vclezq_v:
6508 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
6509 ICmpInst::ICMP_SLE, "vclez");
6510 case NEON::BI__builtin_neon_vcgtz_v:
6511 case NEON::BI__builtin_neon_vcgtzq_v:
6512 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
6513 ICmpInst::ICMP_SGT, "vcgtz");
6514 case NEON::BI__builtin_neon_vcltz_v:
6515 case NEON::BI__builtin_neon_vcltzq_v:
6516 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
6517 ICmpInst::ICMP_SLT, "vcltz");
6518 case NEON::BI__builtin_neon_vcvt_f64_v:
6519 case NEON::BI__builtin_neon_vcvtq_f64_v:
6520 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6521 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
6522 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
6523 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
6524 case NEON::BI__builtin_neon_vcvt_f64_f32: {
6525 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&(static_cast <bool> (Type.getEltType() == NeonTypeFlags
::Float64 && quad && "unexpected vcvt_f64_f32 builtin"
) ? void (0) : __assert_fail ("Type.getEltType() == NeonTypeFlags::Float64 && quad && \"unexpected vcvt_f64_f32 builtin\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 6526, __extension__ __PRETTY_FUNCTION__))
6526 "unexpected vcvt_f64_f32 builtin")(static_cast <bool> (Type.getEltType() == NeonTypeFlags
::Float64 && quad && "unexpected vcvt_f64_f32 builtin"
) ? void (0) : __assert_fail ("Type.getEltType() == NeonTypeFlags::Float64 && quad && \"unexpected vcvt_f64_f32 builtin\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 6526, __extension__ __PRETTY_FUNCTION__))
;
6527 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
6528 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
6529
6530 return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
6531 }
6532 case NEON::BI__builtin_neon_vcvt_f32_f64: {
6533 assert(Type.getEltType() == NeonTypeFlags::Float32 &&(static_cast <bool> (Type.getEltType() == NeonTypeFlags
::Float32 && "unexpected vcvt_f32_f64 builtin") ? void
(0) : __assert_fail ("Type.getEltType() == NeonTypeFlags::Float32 && \"unexpected vcvt_f32_f64 builtin\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 6534, __extension__ __PRETTY_FUNCTION__))
6534 "unexpected vcvt_f32_f64 builtin")(static_cast <bool> (Type.getEltType() == NeonTypeFlags
::Float32 && "unexpected vcvt_f32_f64 builtin") ? void
(0) : __assert_fail ("Type.getEltType() == NeonTypeFlags::Float32 && \"unexpected vcvt_f32_f64 builtin\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 6534, __extension__ __PRETTY_FUNCTION__))
;
6535 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
6536 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
6537
6538 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
6539 }
6540 case NEON::BI__builtin_neon_vcvt_s32_v:
6541 case NEON::BI__builtin_neon_vcvt_u32_v:
6542 case NEON::BI__builtin_neon_vcvt_s64_v:
6543 case NEON::BI__builtin_neon_vcvt_u64_v:
6544 case NEON::BI__builtin_neon_vcvtq_s32_v:
6545 case NEON::BI__builtin_neon_vcvtq_u32_v:
6546 case NEON::BI__builtin_neon_vcvtq_s64_v:
6547 case NEON::BI__builtin_neon_vcvtq_u64_v: {
6548 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
6549 if (usgn)
6550 return Builder.CreateFPToUI(Ops[0], Ty);
6551 return Builder.CreateFPToSI(Ops[0], Ty);
6552 }
6553 case NEON::BI__builtin_neon_vcvta_s32_v:
6554 case NEON::BI__builtin_neon_vcvtaq_s32_v:
6555 case NEON::BI__builtin_neon_vcvta_u32_v:
6556 case NEON::BI__builtin_neon_vcvtaq_u32_v:
6557 case NEON::BI__builtin_neon_vcvta_s64_v:
6558 case NEON::BI__builtin_neon_vcvtaq_s64_v:
6559 case NEON::BI__builtin_neon_vcvta_u64_v:
6560 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
6561 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
6562 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6563 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
6564 }
6565 case NEON::BI__builtin_neon_vcvtm_s32_v:
6566 case NEON::BI__builtin_neon_vcvtmq_s32_v:
6567 case NEON::BI__builtin_neon_vcvtm_u32_v:
6568 case NEON::BI__builtin_neon_vcvtmq_u32_v:
6569 case NEON::BI__builtin_neon_vcvtm_s64_v:
6570 case NEON::BI__builtin_neon_vcvtmq_s64_v:
6571 case NEON::BI__builtin_neon_vcvtm_u64_v:
6572 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6573 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
6574 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6575 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
6576 }
6577 case NEON::BI__builtin_neon_vcvtn_s32_v:
6578 case NEON::BI__builtin_neon_vcvtnq_s32_v:
6579 case NEON::BI__builtin_neon_vcvtn_u32_v:
6580 case NEON::BI__builtin_neon_vcvtnq_u32_v:
6581 case NEON::BI__builtin_neon_vcvtn_s64_v:
6582 case NEON::BI__builtin_neon_vcvtnq_s64_v:
6583 case NEON::BI__builtin_neon_vcvtn_u64_v:
6584 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
6585 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
6586 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6587 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
6588 }
6589 case NEON::BI__builtin_neon_vcvtp_s32_v:
6590 case NEON::BI__builtin_neon_vcvtpq_s32_v:
6591 case NEON::BI__builtin_neon_vcvtp_u32_v:
6592 case NEON::BI__builtin_neon_vcvtpq_u32_v:
6593 case NEON::BI__builtin_neon_vcvtp_s64_v:
6594 case NEON::BI__builtin_neon_vcvtpq_s64_v:
6595 case NEON::BI__builtin_neon_vcvtp_u64_v:
6596 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
6597 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
6598 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6599 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
6600 }
6601 case NEON::BI__builtin_neon_vmulx_v:
6602 case NEON::BI__builtin_neon_vmulxq_v: {
6603 Int = Intrinsic::aarch64_neon_fmulx;
6604 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
6605 }
6606 case NEON::BI__builtin_neon_vmul_lane_v:
6607 case NEON::BI__builtin_neon_vmul_laneq_v: {
6608 // v1f64 vmul_lane should be mapped to Neon scalar mul lane
6609 bool Quad = false;
6610 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
6611 Quad = true;
6612 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6613 llvm::Type *VTy = GetNeonType(this,
6614 NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
6615 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
6616 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
6617 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
6618 return Builder.CreateBitCast(Result, Ty);
6619 }
6620 case NEON::BI__builtin_neon_vnegd_s64:
6621 return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd");
6622 case NEON::BI__builtin_neon_vpmaxnm_v:
6623 case NEON::BI__builtin_neon_vpmaxnmq_v: {
6624 Int = Intrinsic::aarch64_neon_fmaxnmp;
6625 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
6626 }
6627 case NEON::BI__builtin_neon_vpminnm_v:
6628 case NEON::BI__builtin_neon_vpminnmq_v: {
6629 Int = Intrinsic::aarch64_neon_fminnmp;
6630 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
6631 }
6632 case NEON::BI__builtin_neon_vsqrt_v:
6633 case NEON::BI__builtin_neon_vsqrtq_v: {
6634 Int = Intrinsic::sqrt;
6635 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6636 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
6637 }
6638 case NEON::BI__builtin_neon_vrbit_v:
6639 case NEON::BI__builtin_neon_vrbitq_v: {
6640 Int = Intrinsic::aarch64_neon_rbit;
6641 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
6642 }
6643 case NEON::BI__builtin_neon_vaddv_u8:
6644 // FIXME: These are handled by the AArch64 scalar code.
6645 usgn = true;
6646 // FALLTHROUGH
6647 case NEON::BI__builtin_neon_vaddv_s8: {
6648 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6649 Ty = Int32Ty;
6650 VTy = llvm::VectorType::get(Int8Ty, 8);
6651 llvm::Type *Tys[2] = { Ty, VTy };
6652 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6653 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
6654 return Builder.CreateTrunc(Ops[0], Int8Ty);
6655 }
6656 case NEON::BI__builtin_neon_vaddv_u16:
6657 usgn = true;
6658 // FALLTHROUGH
6659 case NEON::BI__builtin_neon_vaddv_s16: {
6660 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6661 Ty = Int32Ty;
6662 VTy = llvm::VectorType::get(Int16Ty, 4);
6663 llvm::Type *Tys[2] = { Ty, VTy };
6664 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6665 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
6666 return Builder.CreateTrunc(Ops[0], Int16Ty);
6667 }
6668 case NEON::BI__builtin_neon_vaddvq_u8:
6669 usgn = true;
6670 // FALLTHROUGH
6671 case NEON::BI__builtin_neon_vaddvq_s8: {
6672 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6673 Ty = Int32Ty;
6674 VTy = llvm::VectorType::get(Int8Ty, 16);
6675 llvm::Type *Tys[2] = { Ty, VTy };
6676 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6677 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
6678 return Builder.CreateTrunc(Ops[0], Int8Ty);
6679 }
6680 case NEON::BI__builtin_neon_vaddvq_u16:
6681 usgn = true;
6682 // FALLTHROUGH
6683 case NEON::BI__builtin_neon_vaddvq_s16: {
6684 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6685 Ty = Int32Ty;
6686 VTy = llvm::VectorType::get(Int16Ty, 8);
6687 llvm::Type *Tys[2] = { Ty, VTy };
6688 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6689 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
6690 return Builder.CreateTrunc(Ops[0], Int16Ty);
6691 }
6692 case NEON::BI__builtin_neon_vmaxv_u8: {
6693 Int = Intrinsic::aarch64_neon_umaxv;
6694 Ty = Int32Ty;
6695 VTy = llvm::VectorType::get(Int8Ty, 8);
6696 llvm::Type *Tys[2] = { Ty, VTy };
6697 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6698 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6699 return Builder.CreateTrunc(Ops[0], Int8Ty);
6700 }
6701 case NEON::BI__builtin_neon_vmaxv_u16: {
6702 Int = Intrinsic::aarch64_neon_umaxv;
6703 Ty = Int32Ty;
6704 VTy = llvm::VectorType::get(Int16Ty, 4);
6705 llvm::Type *Tys[2] = { Ty, VTy };
6706 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6707 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6708 return Builder.CreateTrunc(Ops[0], Int16Ty);
6709 }
6710 case NEON::BI__builtin_neon_vmaxvq_u8: {
6711 Int = Intrinsic::aarch64_neon_umaxv;
6712 Ty = Int32Ty;
6713 VTy = llvm::VectorType::get(Int8Ty, 16);
6714 llvm::Type *Tys[2] = { Ty, VTy };
6715 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6716 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6717 return Builder.CreateTrunc(Ops[0], Int8Ty);
6718 }
6719 case NEON::BI__builtin_neon_vmaxvq_u16: {
6720 Int = Intrinsic::aarch64_neon_umaxv;
6721 Ty = Int32Ty;
6722 VTy = llvm::VectorType::get(Int16Ty, 8);
6723 llvm::Type *Tys[2] = { Ty, VTy };
6724 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6725 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6726 return Builder.CreateTrunc(Ops[0], Int16Ty);
6727 }
6728 case NEON::BI__builtin_neon_vmaxv_s8: {
6729 Int = Intrinsic::aarch64_neon_smaxv;
6730 Ty = Int32Ty;
6731 VTy = llvm::VectorType::get(Int8Ty, 8);
6732 llvm::Type *Tys[2] = { Ty, VTy };
6733 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6734 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6735 return Builder.CreateTrunc(Ops[0], Int8Ty);
6736 }
6737 case NEON::BI__builtin_neon_vmaxv_s16: {
6738 Int = Intrinsic::aarch64_neon_smaxv;
6739 Ty = Int32Ty;
6740 VTy = llvm::VectorType::get(Int16Ty, 4);
6741 llvm::Type *Tys[2] = { Ty, VTy };
6742 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6743 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6744 return Builder.CreateTrunc(Ops[0], Int16Ty);
6745 }
6746 case NEON::BI__builtin_neon_vmaxvq_s8: {
6747 Int = Intrinsic::aarch64_neon_smaxv;
6748 Ty = Int32Ty;
6749 VTy = llvm::VectorType::get(Int8Ty, 16);
6750 llvm::Type *Tys[2] = { Ty, VTy };
6751 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6752 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6753 return Builder.CreateTrunc(Ops[0], Int8Ty);
6754 }
6755 case NEON::BI__builtin_neon_vmaxvq_s16: {
6756 Int = Intrinsic::aarch64_neon_smaxv;
6757 Ty = Int32Ty;
6758 VTy = llvm::VectorType::get(Int16Ty, 8);
6759 llvm::Type *Tys[2] = { Ty, VTy };
6760 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6761 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6762 return Builder.CreateTrunc(Ops[0], Int16Ty);
6763 }
6764 case NEON::BI__builtin_neon_vminv_u8: {
6765 Int = Intrinsic::aarch64_neon_uminv;
6766 Ty = Int32Ty;
6767 VTy = llvm::VectorType::get(Int8Ty, 8);
6768 llvm::Type *Tys[2] = { Ty, VTy };
6769 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6770 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6771 return Builder.CreateTrunc(Ops[0], Int8Ty);
6772 }
6773 case NEON::BI__builtin_neon_vminv_u16: {
6774 Int = Intrinsic::aarch64_neon_uminv;
6775 Ty = Int32Ty;
6776 VTy = llvm::VectorType::get(Int16Ty, 4);
6777 llvm::Type *Tys[2] = { Ty, VTy };
6778 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6779 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6780 return Builder.CreateTrunc(Ops[0], Int16Ty);
6781 }
6782 case NEON::BI__builtin_neon_vminvq_u8: {
6783 Int = Intrinsic::aarch64_neon_uminv;
6784 Ty = Int32Ty;
6785 VTy = llvm::VectorType::get(Int8Ty, 16);
6786 llvm::Type *Tys[2] = { Ty, VTy };
6787 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6788 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6789 return Builder.CreateTrunc(Ops[0], Int8Ty);
6790 }
6791 case NEON::BI__builtin_neon_vminvq_u16: {
6792 Int = Intrinsic::aarch64_neon_uminv;
6793 Ty = Int32Ty;
6794 VTy = llvm::VectorType::get(Int16Ty, 8);
6795 llvm::Type *Tys[2] = { Ty, VTy };
6796 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6797 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6798 return Builder.CreateTrunc(Ops[0], Int16Ty);
6799 }
6800 case NEON::BI__builtin_neon_vminv_s8: {
6801 Int = Intrinsic::aarch64_neon_sminv;
6802 Ty = Int32Ty;
6803 VTy = llvm::VectorType::get(Int8Ty, 8);
6804 llvm::Type *Tys[2] = { Ty, VTy };
6805 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6806 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6807 return Builder.CreateTrunc(Ops[0], Int8Ty);
6808 }
6809 case NEON::BI__builtin_neon_vminv_s16: {
6810 Int = Intrinsic::aarch64_neon_sminv;
6811 Ty = Int32Ty;
6812 VTy = llvm::VectorType::get(Int16Ty, 4);
6813 llvm::Type *Tys[2] = { Ty, VTy };
6814 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6815 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6816 return Builder.CreateTrunc(Ops[0], Int16Ty);
6817 }
6818 case NEON::BI__builtin_neon_vminvq_s8: {
6819 Int = Intrinsic::aarch64_neon_sminv;
6820 Ty = Int32Ty;
6821 VTy = llvm::VectorType::get(Int8Ty, 16);
6822 llvm::Type *Tys[2] = { Ty, VTy };
6823 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6824 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6825 return Builder.CreateTrunc(Ops[0], Int8Ty);
6826 }
6827 case NEON::BI__builtin_neon_vminvq_s16: {
6828 Int = Intrinsic::aarch64_neon_sminv;
6829 Ty = Int32Ty;
6830 VTy = llvm::VectorType::get(Int16Ty, 8);
6831 llvm::Type *Tys[2] = { Ty, VTy };
6832 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6833 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6834 return Builder.CreateTrunc(Ops[0], Int16Ty);
6835 }
6836 case NEON::BI__builtin_neon_vmul_n_f64: {
6837 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6838 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy);
6839 return Builder.CreateFMul(Ops[0], RHS);
6840 }
6841 case NEON::BI__builtin_neon_vaddlv_u8: {
6842 Int = Intrinsic::aarch64_neon_uaddlv;
6843 Ty = Int32Ty;
6844 VTy = llvm::VectorType::get(Int8Ty, 8);
6845 llvm::Type *Tys[2] = { Ty, VTy };
6846 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6847 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6848 return Builder.CreateTrunc(Ops[0], Int16Ty);
6849 }
6850 case NEON::BI__builtin_neon_vaddlv_u16: {
6851 Int = Intrinsic::aarch64_neon_uaddlv;
6852 Ty = Int32Ty;
6853 VTy = llvm::VectorType::get(Int16Ty, 4);
6854 llvm::Type *Tys[2] = { Ty, VTy };
6855 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6856 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6857 }
6858 case NEON::BI__builtin_neon_vaddlvq_u8: {
6859 Int = Intrinsic::aarch64_neon_uaddlv;
6860 Ty = Int32Ty;
6861 VTy = llvm::VectorType::get(Int8Ty, 16);
6862 llvm::Type *Tys[2] = { Ty, VTy };
6863 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6864 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6865 return Builder.CreateTrunc(Ops[0], Int16Ty);
6866 }
6867 case NEON::BI__builtin_neon_vaddlvq_u16: {
6868 Int = Intrinsic::aarch64_neon_uaddlv;
6869 Ty = Int32Ty;
6870 VTy = llvm::VectorType::get(Int16Ty, 8);
6871 llvm::Type *Tys[2] = { Ty, VTy };
6872 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6873 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6874 }
6875 case NEON::BI__builtin_neon_vaddlv_s8: {
6876 Int = Intrinsic::aarch64_neon_saddlv;
6877 Ty = Int32Ty;
6878 VTy = llvm::VectorType::get(Int8Ty, 8);
6879 llvm::Type *Tys[2] = { Ty, VTy };
6880 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6881 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6882 return Builder.CreateTrunc(Ops[0], Int16Ty);
6883 }
6884 case NEON::BI__builtin_neon_vaddlv_s16: {
6885 Int = Intrinsic::aarch64_neon_saddlv;
6886 Ty = Int32Ty;
6887 VTy = llvm::VectorType::get(Int16Ty, 4);
6888 llvm::Type *Tys[2] = { Ty, VTy };
6889 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6890 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6891 }
6892 case NEON::BI__builtin_neon_vaddlvq_s8: {
6893 Int = Intrinsic::aarch64_neon_saddlv;
6894 Ty = Int32Ty;
6895 VTy = llvm::VectorType::get(Int8Ty, 16);
6896 llvm::Type *Tys[2] = { Ty, VTy };
6897 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6898 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6899 return Builder.CreateTrunc(Ops[0], Int16Ty);
6900 }
6901 case NEON::BI__builtin_neon_vaddlvq_s16: {
6902 Int = Intrinsic::aarch64_neon_saddlv;
6903 Ty = Int32Ty;
6904 VTy = llvm::VectorType::get(Int16Ty, 8);
6905 llvm::Type *Tys[2] = { Ty, VTy };
6906 Ops.push_back(EmitScalarExpr(E->getArg(0)));
6907 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6908 }
6909 case NEON::BI__builtin_neon_vsri_n_v:
6910 case NEON::BI__builtin_neon_vsriq_n_v: {
6911 Int = Intrinsic::aarch64_neon_vsri;
6912 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
6913 return EmitNeonCall(Intrin, Ops, "vsri_n");
6914 }
6915 case NEON::BI__builtin_neon_vsli_n_v:
6916 case NEON::BI__builtin_neon_vsliq_n_v: {
6917 Int = Intrinsic::aarch64_neon_vsli;
6918 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
6919 return EmitNeonCall(Intrin, Ops, "vsli_n");
6920 }
6921 case NEON::BI__builtin_neon_vsra_n_v:
6922 case NEON::BI__builtin_neon_vsraq_n_v:
6923 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6924 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
6925 return Builder.CreateAdd(Ops[0], Ops[1]);
6926 case NEON::BI__builtin_neon_vrsra_n_v:
6927 case NEON::BI__builtin_neon_vrsraq_n_v: {
6928 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6929 SmallVector<llvm::Value*,2> TmpOps;
6930 TmpOps.push_back(Ops[1]);
6931 TmpOps.push_back(Ops[2]);
6932 Function* F = CGM.getIntrinsic(Int, Ty);
6933 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true);
6934 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
6935 return Builder.CreateAdd(Ops[0], tmp);
6936 }
6937 // FIXME: Sharing loads & stores with 32-bit is complicated by the absence
6938 // of an Align parameter here.
6939 case NEON::BI__builtin_neon_vld1_x2_v:
6940 case NEON::BI__builtin_neon_vld1q_x2_v:
6941 case NEON::BI__builtin_neon_vld1_x3_v:
6942 case NEON::BI__builtin_neon_vld1q_x3_v:
6943 case NEON::BI__builtin_neon_vld1_x4_v:
6944 case NEON::BI__builtin_neon_vld1q_x4_v: {
6945 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
6946 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
6947 llvm::Type *Tys[2] = { VTy, PTy };
6948 unsigned Int;
6949 switch (BuiltinID) {
6950 case NEON::BI__builtin_neon_vld1_x2_v:
6951 case NEON::BI__builtin_neon_vld1q_x2_v:
6952 Int = Intrinsic::aarch64_neon_ld1x2;
6953 break;
6954 case NEON::BI__builtin_neon_vld1_x3_v:
6955 case NEON::BI__builtin_neon_vld1q_x3_v:
6956 Int = Intrinsic::aarch64_neon_ld1x3;
6957 break;
6958 case NEON::BI__builtin_neon_vld1_x4_v:
6959 case NEON::BI__builtin_neon_vld1q_x4_v:
6960 Int = Intrinsic::aarch64_neon_ld1x4;
6961 break;
6962 }
6963 Function *F = CGM.getIntrinsic(Int, Tys);
6964 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
6965 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6966 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6967 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6968 }
6969 case NEON::BI__builtin_neon_vst1_x2_v:
6970 case NEON::BI__builtin_neon_vst1q_x2_v:
6971 case NEON::BI__builtin_neon_vst1_x3_v:
6972 case NEON::BI__builtin_neon_vst1q_x3_v:
6973 case NEON::BI__builtin_neon_vst1_x4_v:
6974 case NEON::BI__builtin_neon_vst1q_x4_v: {
6975 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
6976 llvm::Type *Tys[2] = { VTy, PTy };
6977 unsigned Int;
6978 switch (BuiltinID) {
6979 case NEON::BI__builtin_neon_vst1_x2_v:
6980 case NEON::BI__builtin_neon_vst1q_x2_v:
6981 Int = Intrinsic::aarch64_neon_st1x2;
6982 break;
6983 case NEON::BI__builtin_neon_vst1_x3_v:
6984 case NEON::BI__builtin_neon_vst1q_x3_v:
6985 Int = Intrinsic::aarch64_neon_st1x3;
6986 break;
6987 case NEON::BI__builtin_neon_vst1_x4_v:
6988 case NEON::BI__builtin_neon_vst1q_x4_v:
6989 Int = Intrinsic::aarch64_neon_st1x4;
6990 break;
6991 }
6992 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6993 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
6994 }
6995 case NEON::BI__builtin_neon_vld1_v:
6996 case NEON::BI__builtin_neon_vld1q_v: {
6997 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
6998 auto Alignment = CharUnits::fromQuantity(
6999 BuiltinID == NEON::BI__builtin_neon_vld1_v ? 8 : 16);
7000 return Builder.CreateAlignedLoad(VTy, Ops[0], Alignment);
7001 }
7002 case NEON::BI__builtin_neon_vst1_v:
7003 case NEON::BI__builtin_neon_vst1q_v:
7004 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
7005 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
7006 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7007 case NEON::BI__builtin_neon_vld1_lane_v:
7008 case NEON::BI__builtin_neon_vld1q_lane_v: {
7009 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7010 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
7011 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7012 auto Alignment = CharUnits::fromQuantity(
7013 BuiltinID == NEON::BI__builtin_neon_vld1_lane_v ? 8 : 16);
7014 Ops[0] =
7015 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
7016 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
7017 }
7018 case NEON::BI__builtin_neon_vld1_dup_v:
7019 case NEON::BI__builtin_neon_vld1q_dup_v: {
7020 Value *V = UndefValue::get(Ty);
7021 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
7022 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7023 auto Alignment = CharUnits::fromQuantity(
7024 BuiltinID == NEON::BI__builtin_neon_vld1_dup_v ? 8 : 16);
7025 Ops[0] =
7026 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
7027 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
7028 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
7029 return EmitNeonSplat(Ops[0], CI);
7030 }
7031 case NEON::BI__builtin_neon_vst1_lane_v:
7032 case NEON::BI__builtin_neon_vst1q_lane_v:
7033 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7034 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
7035 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
7036 return Builder.CreateDefaultAlignedStore(Ops[1],
7037 Builder.CreateBitCast(Ops[0], Ty));
7038 case NEON::BI__builtin_neon_vld2_v:
7039 case NEON::BI__builtin_neon_vld2q_v: {
7040 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
7041 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
7042 llvm::Type *Tys[2] = { VTy, PTy };
7043 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
7044 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
7045 Ops[0] = Builder.CreateBitCast(Ops[0],
7046 llvm::PointerType::getUnqual(Ops[1]->getType()));
7047 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7048 }
7049 case NEON::BI__builtin_neon_vld3_v:
7050 case NEON::BI__builtin_neon_vld3q_v: {
7051 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
7052 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
7053 llvm::Type *Tys[2] = { VTy, PTy };
7054 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
7055 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
7056 Ops[0] = Builder.CreateBitCast(Ops[0],
7057 llvm::PointerType::getUnqual(Ops[1]->getType()));
7058 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7059 }
7060 case NEON::BI__builtin_neon_vld4_v:
7061 case NEON::BI__builtin_neon_vld4q_v: {
7062 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
7063 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
7064 llvm::Type *Tys[2] = { VTy, PTy };
7065 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
7066 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
7067 Ops[0] = Builder.CreateBitCast(Ops[0],
7068 llvm::PointerType::getUnqual(Ops[1]->getType()));
7069 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7070 }
7071 case NEON::BI__builtin_neon_vld2_dup_v:
7072 case NEON::BI__builtin_neon_vld2q_dup_v: {
7073 llvm::Type *PTy =
7074 llvm::PointerType::getUnqual(VTy->getElementType());
7075 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
7076 llvm::Type *Tys[2] = { VTy, PTy };
7077 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
7078 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
7079 Ops[0] = Builder.CreateBitCast(Ops[0],
7080 llvm::PointerType::getUnqual(Ops[1]->getType()));
7081 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7082 }
7083 case NEON::BI__builtin_neon_vld3_dup_v:
7084 case NEON::BI__builtin_neon_vld3q_dup_v: {
7085 llvm::Type *PTy =
7086 llvm::PointerType::getUnqual(VTy->getElementType());
7087 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
7088 llvm::Type *Tys[2] = { VTy, PTy };
7089 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
7090 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
7091 Ops[0] = Builder.CreateBitCast(Ops[0],
7092 llvm::PointerType::getUnqual(Ops[1]->getType()));
7093 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7094 }
7095 case NEON::BI__builtin_neon_vld4_dup_v:
7096 case NEON::BI__builtin_neon_vld4q_dup_v: {
7097 llvm::Type *PTy =
7098 llvm::PointerType::getUnqual(VTy->getElementType());
7099 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
7100 llvm::Type *Tys[2] = { VTy, PTy };
7101 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
7102 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
7103 Ops[0] = Builder.CreateBitCast(Ops[0],
7104 llvm::PointerType::getUnqual(Ops[1]->getType()));
7105 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7106 }
7107 case NEON::BI__builtin_neon_vld2_lane_v:
7108 case NEON::BI__builtin_neon_vld2q_lane_v: {
7109 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
7110 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
7111 Ops.push_back(Ops[1]);
7112 Ops.erase(Ops.begin()+1);
7113 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7114 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7115 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
7116 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
7117 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
7118 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7119 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7120 }
7121 case NEON::BI__builtin_neon_vld3_lane_v:
7122 case NEON::BI__builtin_neon_vld3q_lane_v: {
7123 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
7124 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
7125 Ops.push_back(Ops[1]);
7126 Ops.erase(Ops.begin()+1);
7127 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7128 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7129 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
7130 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
7131 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
7132 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
7133 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7134 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7135 }
7136 case NEON::BI__builtin_neon_vld4_lane_v:
7137 case NEON::BI__builtin_neon_vld4q_lane_v: {
7138 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
7139 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
7140 Ops.push_back(Ops[1]);
7141 Ops.erase(Ops.begin()+1);
7142 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7143 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7144 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
7145 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
7146 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
7147 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane");
7148 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
7149 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7150 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7151 }
7152 case NEON::BI__builtin_neon_vst2_v:
7153 case NEON::BI__builtin_neon_vst2q_v: {
7154 Ops.push_back(Ops[0]);
7155 Ops.erase(Ops.begin());
7156 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
7157 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
7158 Ops, "");
7159 }
7160 case NEON::BI__builtin_neon_vst2_lane_v:
7161 case NEON::BI__builtin_neon_vst2q_lane_v: {
7162 Ops.push_back(Ops[0]);
7163 Ops.erase(Ops.begin());
7164 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
7165 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
7166 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
7167 Ops, "");
7168 }
7169 case NEON::BI__builtin_neon_vst3_v:
7170 case NEON::BI__builtin_neon_vst3q_v: {
7171 Ops.push_back(Ops[0]);
7172 Ops.erase(Ops.begin());
7173 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
7174 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
7175 Ops, "");
7176 }
7177 case NEON::BI__builtin_neon_vst3_lane_v:
7178 case NEON::BI__builtin_neon_vst3q_lane_v: {
7179 Ops.push_back(Ops[0]);
7180 Ops.erase(Ops.begin());
7181 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
7182 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
7183 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
7184 Ops, "");
7185 }
7186 case NEON::BI__builtin_neon_vst4_v:
7187 case NEON::BI__builtin_neon_vst4q_v: {
7188 Ops.push_back(Ops[0]);
7189 Ops.erase(Ops.begin());
7190 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
7191 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
7192 Ops, "");
7193 }
7194 case NEON::BI__builtin_neon_vst4_lane_v:
7195 case NEON::BI__builtin_neon_vst4q_lane_v: {
7196 Ops.push_back(Ops[0]);
7197 Ops.erase(Ops.begin());
7198 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
7199 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
7200 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
7201 Ops, "");
7202 }
7203 case NEON::BI__builtin_neon_vtrn_v:
7204 case NEON::BI__builtin_neon_vtrnq_v: {
7205 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
7206 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7207 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7208 Value *SV = nullptr;
7209
7210 for (unsigned vi = 0; vi != 2; ++vi) {
7211 SmallVector<uint32_t, 16> Indices;
7212 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
7213 Indices.push_back(i+vi);
7214 Indices.push_back(i+e+vi);
7215 }
7216 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7217 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
7218 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7219 }
7220 return SV;
7221 }
7222 case NEON::BI__builtin_neon_vuzp_v:
7223 case NEON::BI__builtin_neon_vuzpq_v: {
7224 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
7225 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7226 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7227 Value *SV = nullptr;
7228
7229 for (unsigned vi = 0; vi != 2; ++vi) {
7230 SmallVector<uint32_t, 16> Indices;
7231 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7232 Indices.push_back(2*i+vi);
7233
7234 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7235 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
7236 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7237 }
7238 return SV;
7239 }
7240 case NEON::BI__builtin_neon_vzip_v:
7241 case NEON::BI__builtin_neon_vzipq_v: {
7242 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
7243 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7244 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7245 Value *SV = nullptr;
7246
7247 for (unsigned vi = 0; vi != 2; ++vi) {
7248 SmallVector<uint32_t, 16> Indices;
7249 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
7250 Indices.push_back((i + vi*e) >> 1);
7251 Indices.push_back(((i + vi*e) >> 1)+e);
7252 }
7253 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7254 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
7255 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7256 }
7257 return SV;
7258 }
7259 case NEON::BI__builtin_neon_vqtbl1q_v: {
7260 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
7261 Ops, "vtbl1");
7262 }
7263 case NEON::BI__builtin_neon_vqtbl2q_v: {
7264 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
7265 Ops, "vtbl2");
7266 }
7267 case NEON::BI__builtin_neon_vqtbl3q_v: {
7268 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
7269 Ops, "vtbl3");
7270 }
7271 case NEON::BI__builtin_neon_vqtbl4q_v: {
7272 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
7273 Ops, "vtbl4");
7274 }
7275 case NEON::BI__builtin_neon_vqtbx1q_v: {
7276 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
7277 Ops, "vtbx1");
7278 }
7279 case NEON::BI__builtin_neon_vqtbx2q_v: {
7280 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
7281 Ops, "vtbx2");
7282 }
7283 case NEON::BI__builtin_neon_vqtbx3q_v: {
7284 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
7285 Ops, "vtbx3");
7286 }
7287 case NEON::BI__builtin_neon_vqtbx4q_v: {
7288 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
7289 Ops, "vtbx4");
7290 }
7291 case NEON::BI__builtin_neon_vsqadd_v:
7292 case NEON::BI__builtin_neon_vsqaddq_v: {
7293 Int = Intrinsic::aarch64_neon_usqadd;
7294 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
7295 }
7296 case NEON::BI__builtin_neon_vuqadd_v:
7297 case NEON::BI__builtin_neon_vuqaddq_v: {
7298 Int = Intrinsic::aarch64_neon_suqadd;
7299 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
7300 }
7301 }
7302}
7303
7304llvm::Value *CodeGenFunction::
7305BuildVector(ArrayRef<llvm::Value*> Ops) {
7306 assert((Ops.size() & (Ops.size() - 1)) == 0 &&(static_cast <bool> ((Ops.size() & (Ops.size() - 1)
) == 0 && "Not a power-of-two sized vector!") ? void (
0) : __assert_fail ("(Ops.size() & (Ops.size() - 1)) == 0 && \"Not a power-of-two sized vector!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 7307, __extension__ __PRETTY_FUNCTION__))
7307 "Not a power-of-two sized vector!")(static_cast <bool> ((Ops.size() & (Ops.size() - 1)
) == 0 && "Not a power-of-two sized vector!") ? void (
0) : __assert_fail ("(Ops.size() & (Ops.size() - 1)) == 0 && \"Not a power-of-two sized vector!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 7307, __extension__ __PRETTY_FUNCTION__))
;
7308 bool AllConstants = true;
7309 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
7310 AllConstants &= isa<Constant>(Ops[i]);
7311
7312 // If this is a constant vector, create a ConstantVector.
7313 if (AllConstants) {
7314 SmallVector<llvm::Constant*, 16> CstOps;
7315 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
7316 CstOps.push_back(cast<Constant>(Ops[i]));
7317 return llvm::ConstantVector::get(CstOps);
7318 }
7319
7320 // Otherwise, insertelement the values to build the vector.
7321 Value *Result =
7322 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
7323
7324 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
7325 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
7326
7327 return Result;
7328}
7329
7330// Convert the mask from an integer type to a vector of i1.
7331static Value *getMaskVecValue(CodeGenFunction &CGF, Value *Mask,
7332 unsigned NumElts) {
7333
7334 llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.Builder.getInt1Ty(),
7335 cast<IntegerType>(Mask->getType())->getBitWidth());
7336 Value *MaskVec = CGF.Builder.CreateBitCast(Mask, MaskTy);
7337
7338 // If we have less than 8 elements, then the starting mask was an i8 and
7339 // we need to extract down to the right number of elements.
7340 if (NumElts < 8) {
7341 uint32_t Indices[4];
7342 for (unsigned i = 0; i != NumElts; ++i)
7343 Indices[i] = i;
7344 MaskVec = CGF.Builder.CreateShuffleVector(MaskVec, MaskVec,
7345 makeArrayRef(Indices, NumElts),
7346 "extract");
7347 }
7348 return MaskVec;
7349}
7350
7351static Value *EmitX86MaskedStore(CodeGenFunction &CGF,
7352 SmallVectorImpl<Value *> &Ops,
7353 unsigned Align) {
7354 // Cast the pointer to right type.
7355 Ops[0] = CGF.Builder.CreateBitCast(Ops[0],
7356 llvm::PointerType::getUnqual(Ops[1]->getType()));
7357
7358 // If the mask is all ones just emit a regular store.
7359 if (const auto *C = dyn_cast<Constant>(Ops[2]))
7360 if (C->isAllOnesValue())
7361 return CGF.Builder.CreateAlignedStore(Ops[1], Ops[0], Align);
7362
7363 Value *MaskVec = getMaskVecValue(CGF, Ops[2],
7364 Ops[1]->getType()->getVectorNumElements());
7365
7366 return CGF.Builder.CreateMaskedStore(Ops[1], Ops[0], Align, MaskVec);
7367}
7368
7369static Value *EmitX86MaskedLoad(CodeGenFunction &CGF,
7370 SmallVectorImpl<Value *> &Ops, unsigned Align) {
7371 // Cast the pointer to right type.
7372 Ops[0] = CGF.Builder.CreateBitCast(Ops[0],
7373 llvm::PointerType::getUnqual(Ops[1]->getType()));
7374
7375 // If the mask is all ones just emit a regular store.
7376 if (const auto *C = dyn_cast<Constant>(Ops[2]))
7377 if (C->isAllOnesValue())
7378 return CGF.Builder.CreateAlignedLoad(Ops[0], Align);
7379
7380 Value *MaskVec = getMaskVecValue(CGF, Ops[2],
7381 Ops[1]->getType()->getVectorNumElements());
7382
7383 return CGF.Builder.CreateMaskedLoad(Ops[0], Align, MaskVec, Ops[1]);
7384}
7385
7386static Value *EmitX86SubVectorBroadcast(CodeGenFunction &CGF,
7387 SmallVectorImpl<Value *> &Ops,
7388 llvm::Type *DstTy,
7389 unsigned SrcSizeInBits,
7390 unsigned Align) {
7391 // Load the subvector.
7392 Ops[0] = CGF.Builder.CreateAlignedLoad(Ops[0], Align);
7393
7394 // Create broadcast mask.
7395 unsigned NumDstElts = DstTy->getVectorNumElements();
7396 unsigned NumSrcElts = SrcSizeInBits / DstTy->getScalarSizeInBits();
7397
7398 SmallVector<uint32_t, 8> Mask;
7399 for (unsigned i = 0; i != NumDstElts; i += NumSrcElts)
7400 for (unsigned j = 0; j != NumSrcElts; ++j)
7401 Mask.push_back(j);
7402
7403 return CGF.Builder.CreateShuffleVector(Ops[0], Ops[0], Mask, "subvecbcst");
7404}
7405
7406static Value *EmitX86Select(CodeGenFunction &CGF,
7407 Value *Mask, Value *Op0, Value *Op1) {
7408
7409 // If the mask is all ones just return first argument.
7410 if (const auto *C = dyn_cast<Constant>(Mask))
7411 if (C->isAllOnesValue())
7412 return Op0;
7413
7414 Mask = getMaskVecValue(CGF, Mask, Op0->getType()->getVectorNumElements());
7415
7416 return CGF.Builder.CreateSelect(Mask, Op0, Op1);
7417}
7418
7419static Value *EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC,
7420 bool Signed, SmallVectorImpl<Value *> &Ops) {
7421 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
1
Calling 'Type::getVectorNumElements'
27
Returning from 'Type::getVectorNumElements'
28
'NumElts' initialized here
7422 Value *Cmp;
7423
7424 if (CC == 3) {
29
Assuming 'CC' is not equal to 3
30
Taking false branch
7425 Cmp = Constant::getNullValue(
7426 llvm::VectorType::get(CGF.Builder.getInt1Ty(), NumElts));
7427 } else if (CC == 7) {
31
Assuming 'CC' is equal to 7
32
Taking true branch
7428 Cmp = Constant::getAllOnesValue(
7429 llvm::VectorType::get(CGF.Builder.getInt1Ty(), NumElts));
7430 } else {
7431 ICmpInst::Predicate Pred;
7432 switch (CC) {
7433 default: llvm_unreachable("Unknown condition code")::llvm::llvm_unreachable_internal("Unknown condition code", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 7433)
;
7434 case 0: Pred = ICmpInst::ICMP_EQ; break;
7435 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
7436 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
7437 case 4: Pred = ICmpInst::ICMP_NE; break;
7438 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
7439 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
7440 }
7441 Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]);
7442 }
7443
7444 const auto *C = dyn_cast<Constant>(Ops.back());
7445 if (!C || !C->isAllOnesValue())
33
Assuming 'C' is non-null
34
Assuming the condition is false
35
Taking false branch
7446 Cmp = CGF.Builder.CreateAnd(Cmp, getMaskVecValue(CGF, Ops.back(), NumElts));
7447
7448 if (NumElts < 8) {
36
Assuming 'NumElts' is < 8
37
Taking true branch
7449 uint32_t Indices[8];
7450 for (unsigned i = 0; i != NumElts; ++i)
38
Assuming 'i' is equal to 'NumElts'
39
Loop condition is false. Execution continues on line 7452
7451 Indices[i] = i;
7452 for (unsigned i = NumElts; i != 8; ++i)
40
Loop condition is true. Entering loop body
7453 Indices[i] = i % NumElts + NumElts;
41
Division by zero
7454 Cmp = CGF.Builder.CreateShuffleVector(
7455 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
7456 }
7457 return CGF.Builder.CreateBitCast(Cmp,
7458 IntegerType::get(CGF.getLLVMContext(),
7459 std::max(NumElts, 8U)));
7460}
7461
7462static Value *EmitX86Abs(CodeGenFunction &CGF, ArrayRef<Value *> Ops) {
7463
7464 llvm::Type *Ty = Ops[0]->getType();
7465 Value *Zero = llvm::Constant::getNullValue(Ty);
7466 Value *Sub = CGF.Builder.CreateSub(Zero, Ops[0]);
7467 Value *Cmp = CGF.Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Zero);
7468 Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Sub);
7469 if (Ops.size() == 1)
7470 return Res;
7471 return EmitX86Select(CGF, Ops[2], Res, Ops[1]);
7472}
7473
7474static Value *EmitX86MinMax(CodeGenFunction &CGF, ICmpInst::Predicate Pred,
7475 ArrayRef<Value *> Ops) {
7476 Value *Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]);
7477 Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
7478
7479 if (Ops.size() == 2)
7480 return Res;
7481
7482 assert(Ops.size() == 4)(static_cast <bool> (Ops.size() == 4) ? void (0) : __assert_fail
("Ops.size() == 4", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 7482, __extension__ __PRETTY_FUNCTION__))
;
7483 return EmitX86Select(CGF, Ops[3], Res, Ops[2]);
7484}
7485
7486static Value *EmitX86SExtMask(CodeGenFunction &CGF, Value *Op,
7487 llvm::Type *DstTy) {
7488 unsigned NumberOfElements = DstTy->getVectorNumElements();
7489 Value *Mask = getMaskVecValue(CGF, Op, NumberOfElements);
7490 return CGF.Builder.CreateSExt(Mask, DstTy, "vpmovm2");
7491}
7492
7493Value *CodeGenFunction::EmitX86CpuIs(const CallExpr *E) {
7494 const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
7495 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
7496 return EmitX86CpuIs(CPUStr);
7497}
7498
7499Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
7500
7501 llvm::Type *Int32Ty = Builder.getInt32Ty();
7502
7503 // Matching the struct layout from the compiler-rt/libgcc structure that is
7504 // filled in:
7505 // unsigned int __cpu_vendor;
7506 // unsigned int __cpu_type;
7507 // unsigned int __cpu_subtype;
7508 // unsigned int __cpu_features[1];
7509 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
7510 llvm::ArrayType::get(Int32Ty, 1));
7511
7512 // Grab the global __cpu_model.
7513 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
7514
7515 // Calculate the index needed to access the correct field based on the
7516 // range. Also adjust the expected value.
7517 unsigned Index;
7518 unsigned Value;
7519 std::tie(Index, Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
7520#define X86_VENDOR(ENUM, STRING) \
7521 .Case(STRING, {0u, static_cast<unsigned>(llvm::X86::ENUM)})
7522#define X86_CPU_TYPE_COMPAT_WITH_ALIAS(ARCHNAME, ENUM, STR, ALIAS) \
7523 .Cases(STR, ALIAS, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
7524#define X86_CPU_TYPE_COMPAT(ARCHNAME, ENUM, STR) \
7525 .Case(STR, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
7526#define X86_CPU_SUBTYPE_COMPAT(ARCHNAME, ENUM, STR) \
7527 .Case(STR, {2u, static_cast<unsigned>(llvm::X86::ENUM)})
7528#include "llvm/Support/X86TargetParser.def"
7529 .Default({0, 0});
7530 assert(Value != 0 && "Invalid CPUStr passed to CpuIs")(static_cast <bool> (Value != 0 && "Invalid CPUStr passed to CpuIs"
) ? void (0) : __assert_fail ("Value != 0 && \"Invalid CPUStr passed to CpuIs\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 7530, __extension__ __PRETTY_FUNCTION__))
;
7531
7532 // Grab the appropriate field from __cpu_model.
7533 llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0),
7534 ConstantInt::get(Int32Ty, Index)};
7535 llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs);
7536 CpuValue = Builder.CreateAlignedLoad(CpuValue, CharUnits::fromQuantity(4));
7537
7538 // Check the value of the field against the requested value.
7539 return Builder.CreateICmpEQ(CpuValue,
7540 llvm::ConstantInt::get(Int32Ty, Value));
7541}
7542
7543Value *CodeGenFunction::EmitX86CpuSupports(const CallExpr *E) {
7544 const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts();
7545 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
7546 return EmitX86CpuSupports(FeatureStr);
7547}
7548
7549Value *CodeGenFunction::EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs) {
7550 // TODO: When/if this becomes more than x86 specific then use a TargetInfo
7551 // based mapping.
7552 // Processor features and mapping to processor feature value.
7553 enum X86Features {
7554 CMOV = 0,
7555 MMX,
7556 POPCNT,
7557 SSE,
7558 SSE2,
7559 SSE3,
7560 SSSE3,
7561 SSE4_1,
7562 SSE4_2,
7563 AVX,
7564 AVX2,
7565 SSE4_A,
7566 FMA4,
7567 XOP,
7568 FMA,
7569 AVX512F,
7570 BMI,
7571 BMI2,
7572 AES,
7573 PCLMUL,
7574 AVX512VL,
7575 AVX512BW,
7576 AVX512DQ,
7577 AVX512CD,
7578 AVX512ER,
7579 AVX512PF,
7580 AVX512VBMI,
7581 AVX512IFMA,
7582 AVX5124VNNIW,
7583 AVX5124FMAPS,
7584 AVX512VPOPCNTDQ,
7585 MAX
7586 };
7587
7588 uint32_t FeaturesMask = 0;
7589
7590 for (const StringRef &FeatureStr : FeatureStrs) {
7591 X86Features Feature =
7592 StringSwitch<X86Features>(FeatureStr)
7593 .Case("cmov", X86Features::CMOV)
7594 .Case("mmx", X86Features::MMX)
7595 .Case("popcnt", X86Features::POPCNT)
7596 .Case("sse", X86Features::SSE)
7597 .Case("sse2", X86Features::SSE2)
7598 .Case("sse3", X86Features::SSE3)
7599 .Case("ssse3", X86Features::SSSE3)
7600 .Case("sse4.1", X86Features::SSE4_1)
7601 .Case("sse4.2", X86Features::SSE4_2)
7602 .Case("avx", X86Features::AVX)
7603 .Case("avx2", X86Features::AVX2)
7604 .Case("sse4a", X86Features::SSE4_A)
7605 .Case("fma4", X86Features::FMA4)
7606 .Case("xop", X86Features::XOP)
7607 .Case("fma", X86Features::FMA)
7608 .Case("avx512f", X86Features::AVX512F)
7609 .Case("bmi", X86Features::BMI)
7610 .Case("bmi2", X86Features::BMI2)
7611 .Case("aes", X86Features::AES)
7612 .Case("pclmul", X86Features::PCLMUL)
7613 .Case("avx512vl", X86Features::AVX512VL)
7614 .Case("avx512bw", X86Features::AVX512BW)
7615 .Case("avx512dq", X86Features::AVX512DQ)
7616 .Case("avx512cd", X86Features::AVX512CD)
7617 .Case("avx512er", X86Features::AVX512ER)
7618 .Case("avx512pf", X86Features::AVX512PF)
7619 .Case("avx512vbmi", X86Features::AVX512VBMI)
7620 .Case("avx512ifma", X86Features::AVX512IFMA)
7621 .Case("avx5124vnniw", X86Features::AVX5124VNNIW)
7622 .Case("avx5124fmaps", X86Features::AVX5124FMAPS)
7623 .Case("avx512vpopcntdq", X86Features::AVX512VPOPCNTDQ)
7624 .Default(X86Features::MAX);
7625 assert(Feature != X86Features::MAX && "Invalid feature!")(static_cast <bool> (Feature != X86Features::MAX &&
"Invalid feature!") ? void (0) : __assert_fail ("Feature != X86Features::MAX && \"Invalid feature!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 7625, __extension__ __PRETTY_FUNCTION__))
;
7626 FeaturesMask |= (1U << Feature);
7627 }
7628
7629 // Matching the struct layout from the compiler-rt/libgcc structure that is
7630 // filled in:
7631 // unsigned int __cpu_vendor;
7632 // unsigned int __cpu_type;
7633 // unsigned int __cpu_subtype;
7634 // unsigned int __cpu_features[1];
7635 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
7636 llvm::ArrayType::get(Int32Ty, 1));
7637
7638 // Grab the global __cpu_model.
7639 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
7640
7641 // Grab the first (0th) element from the field __cpu_features off of the
7642 // global in the struct STy.
7643 Value *Idxs[] = {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, 3),
7644 ConstantInt::get(Int32Ty, 0)};
7645 Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
7646 Value *Features =
7647 Builder.CreateAlignedLoad(CpuFeatures, CharUnits::fromQuantity(4));
7648
7649 // Check the value of the bit corresponding to the feature requested.
7650 Value *Bitset = Builder.CreateAnd(
7651 Features, llvm::ConstantInt::get(Int32Ty, FeaturesMask));
7652 return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0));
7653}
7654
7655Value *CodeGenFunction::EmitX86CpuInit() {
7656 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
7657 /*Variadic*/ false);
7658 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, "__cpu_indicator_init");
7659 return Builder.CreateCall(Func);
7660}
7661
7662Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
7663 const CallExpr *E) {
7664 if (BuiltinID == X86::BI__builtin_cpu_is)
7665 return EmitX86CpuIs(E);
7666 if (BuiltinID == X86::BI__builtin_cpu_supports)
7667 return EmitX86CpuSupports(E);
7668 if (BuiltinID == X86::BI__builtin_cpu_init)
7669 return EmitX86CpuInit();
7670
7671 SmallVector<Value*, 4> Ops;
7672
7673 // Find out if any arguments are required to be integer constant expressions.
7674 unsigned ICEArguments = 0;
7675 ASTContext::GetBuiltinTypeError Error;
7676 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
7677 assert(Error == ASTContext::GE_None && "Should not codegen an error")(static_cast <bool> (Error == ASTContext::GE_None &&
"Should not codegen an error") ? void (0) : __assert_fail ("Error == ASTContext::GE_None && \"Should not codegen an error\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 7677, __extension__ __PRETTY_FUNCTION__))
;
7678
7679 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
7680 // If this is a normal argument, just emit it as a scalar.
7681 if ((ICEArguments & (1 << i)) == 0) {
7682 Ops.push_back(EmitScalarExpr(E->getArg(i)));
7683 continue;
7684 }
7685
7686 // If this is required to be a constant, constant fold it so that we know
7687 // that the generated intrinsic gets a ConstantInt.
7688 llvm::APSInt Result;
7689 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
7690 assert(IsConst && "Constant arg isn't actually constant?")(static_cast <bool> (IsConst && "Constant arg isn't actually constant?"
) ? void (0) : __assert_fail ("IsConst && \"Constant arg isn't actually constant?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 7690, __extension__ __PRETTY_FUNCTION__))
; (void)IsConst;
7691 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
7692 }
7693
7694 // These exist so that the builtin that takes an immediate can be bounds
7695 // checked by clang to avoid passing bad immediates to the backend. Since
7696 // AVX has a larger immediate than SSE we would need separate builtins to
7697 // do the different bounds checking. Rather than create a clang specific
7698 // SSE only builtin, this implements eight separate builtins to match gcc
7699 // implementation.
7700 auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) {
7701 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm));
7702 llvm::Function *F = CGM.getIntrinsic(ID);
7703 return Builder.CreateCall(F, Ops);
7704 };
7705
7706 // For the vector forms of FP comparisons, translate the builtins directly to
7707 // IR.
7708 // TODO: The builtins could be removed if the SSE header files used vector
7709 // extension comparisons directly (vector ordered/unordered may need
7710 // additional support via __builtin_isnan()).
7711 auto getVectorFCmpIR = [this, &Ops](CmpInst::Predicate Pred) {
7712 Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
7713 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
7714 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
7715 Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
7716 return Builder.CreateBitCast(Sext, FPVecTy);
7717 };
7718
7719 switch (BuiltinID) {
7720 default: return nullptr;
7721 case X86::BI_mm_prefetch: {
7722 Value *Address = Ops[0];
7723 Value *RW = ConstantInt::get(Int32Ty, 0);
7724 Value *Locality = Ops[1];
7725 Value *Data = ConstantInt::get(Int32Ty, 1);
7726 Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
7727 return Builder.CreateCall(F, {Address, RW, Locality, Data});
7728 }
7729 case X86::BI_mm_clflush: {
7730 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
7731 Ops[0]);
7732 }
7733 case X86::BI_mm_lfence: {
7734 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence));
7735 }
7736 case X86::BI_mm_mfence: {
7737 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence));
7738 }
7739 case X86::BI_mm_sfence: {
7740 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence));
7741 }
7742 case X86::BI_mm_pause: {
7743 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause));
7744 }
7745 case X86::BI__rdtsc: {
7746 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc));
7747 }
7748 case X86::BI__builtin_ia32_undef128:
7749 case X86::BI__builtin_ia32_undef256:
7750 case X86::BI__builtin_ia32_undef512:
7751 // The x86 definition of "undef" is not the same as the LLVM definition
7752 // (PR32176). We leave optimizing away an unnecessary zero constant to the
7753 // IR optimizer and backend.
7754 // TODO: If we had a "freeze" IR instruction to generate a fixed undef
7755 // value, we should use that here instead of a zero.
7756 return llvm::Constant::getNullValue(ConvertType(E->getType()));
7757 case X86::BI__builtin_ia32_vec_init_v8qi:
7758 case X86::BI__builtin_ia32_vec_init_v4hi:
7759 case X86::BI__builtin_ia32_vec_init_v2si:
7760 return Builder.CreateBitCast(BuildVector(Ops),
7761 llvm::Type::getX86_MMXTy(getLLVMContext()));
7762 case X86::BI__builtin_ia32_vec_ext_v2si:
7763 return Builder.CreateExtractElement(Ops[0],
7764 llvm::ConstantInt::get(Ops[1]->getType(), 0));
7765 case X86::BI_mm_setcsr:
7766 case X86::BI__builtin_ia32_ldmxcsr: {
7767 Address Tmp = CreateMemTemp(E->getArg(0)->getType());
7768 Builder.CreateStore(Ops[0], Tmp);
7769 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
7770 Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
7771 }
7772 case X86::BI_mm_getcsr:
7773 case X86::BI__builtin_ia32_stmxcsr: {
7774 Address Tmp = CreateMemTemp(E->getType());
7775 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
7776 Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
7777 return Builder.CreateLoad(Tmp, "stmxcsr");
7778 }
7779 case X86::BI__builtin_ia32_xsave:
7780 case X86::BI__builtin_ia32_xsave64:
7781 case X86::BI__builtin_ia32_xrstor:
7782 case X86::BI__builtin_ia32_xrstor64:
7783 case X86::BI__builtin_ia32_xsaveopt:
7784 case X86::BI__builtin_ia32_xsaveopt64:
7785 case X86::BI__builtin_ia32_xrstors:
7786 case X86::BI__builtin_ia32_xrstors64:
7787 case X86::BI__builtin_ia32_xsavec:
7788 case X86::BI__builtin_ia32_xsavec64:
7789 case X86::BI__builtin_ia32_xsaves:
7790 case X86::BI__builtin_ia32_xsaves64: {
7791 Intrinsic::ID ID;
7792#define INTRINSIC_X86_XSAVE_ID(NAME) \
7793 case X86::BI__builtin_ia32_##NAME: \
7794 ID = Intrinsic::x86_##NAME; \
7795 break
7796 switch (BuiltinID) {
7797 default: llvm_unreachable("Unsupported intrinsic!")::llvm::llvm_unreachable_internal("Unsupported intrinsic!", "/build/llvm-toolchain-snapshot-6.0~svn318601/tools/clang/lib/CodeGen/CGBuiltin.cpp"
, 7797)
;
7798 INTRINSIC_X86_XSAVE_ID(xsave);
7799 INTRINSIC_X86_XSAVE_ID(xsave64);
7800 INTRINSIC_X86_XSAVE_ID(xrstor);
7801 INTRINSIC_X86_XSAVE_ID(xrstor64);
7802 INTRINSIC_X86_XSAVE_ID(xsaveopt);
7803 INTRINSIC_X86_XSAVE_ID(xsaveopt64);
7804 INTRINSIC_X86_XSAVE_ID(xrstors);
7805 INTRINSIC_X86_XSAVE_ID(xrstors64);
7806 INTRINSIC_X86_XSAVE_ID(xsavec);
7807 INTRINSIC_X86_XSAVE_ID(xsavec64);
7808 INTRINSIC_X86_XSAVE_ID(xsaves);
7809 INTRINSIC_X86_XSAVE_ID(xsaves64);
7810 }
7811#undef INTRINSIC_X86_XSAVE_ID
7812 Value *Mhi = Builder.CreateTrunc(
7813 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
7814 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
7815 Ops[1] = Mhi;
7816 Ops.push_back(Mlo);
7817 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
7818 }
7819 case X86::BI__builtin_ia32_storedqudi128_mask:
7820 case X86::BI__builtin_ia32_storedqusi128_mask:
7821 case X86::BI__builtin_ia32_storedquhi128_mask:
7822 case X86::BI__builtin_ia32_storedquqi128_mask:
7823 case X86::BI__builtin_ia32_storeupd128_mask:
7824 case X86::BI__builtin_ia32_storeups128_mask:
7825 case X86::BI__builtin_ia32_storedqudi256_mask:
7826 case X86::BI__builtin_ia32_storedqusi256_mask:
7827 case X86::BI__builtin_ia32_storedquhi256_mask:
7828 case X86::BI__builtin_ia32_storedquqi256_mask:
7829 case X86::BI__builtin_ia32_storeupd256_mask:
7830 case X86::BI__builtin_ia32_storeups256_mask:
7831 case X86::BI__builtin_ia32_storedqudi512_mask:
7832 case X86::BI__builtin_ia32_storedqusi512_mask:
7833 case X86::BI__builtin_ia32_storedquhi512_mask:
7834 case X86::BI__builtin_ia32_storedquqi512_mask:
7835 case X86::BI__builtin_ia32_storeupd512_mask:
7836 case X86::BI__builtin_ia32_storeups512_mask:
7837 return EmitX86MaskedStore(*this, Ops, 1);
7838
7839 case X86::BI__builtin_ia32_storess128_mask:
7840 case X86::BI__builtin_ia32_storesd128_mask: {
7841 return EmitX86MaskedStore(*this, Ops, 16);
7842 }
7843 case X86::BI__builtin_ia32_vpopcntd_512:
7844 case X86::BI__builtin_ia32_vpopcntq_512: {
7845 llvm::Type *ResultType = ConvertType(E->getType());
7846 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
7847 return Builder.CreateCall(F, Ops);
7848 }
7849 case X86::BI__builtin_ia32_cvtmask2b128:
7850 case X86::BI__builtin_ia32_cvtmask2b256:
7851 case X86::BI__builtin_ia32_cvtmask2b512: