LLVM 24.0.0git
SPIRVLegalizePointerCast.cpp
Go to the documentation of this file.
1//===-- SPIRVLegalizePointerCast.cpp ----------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The LLVM IR has multiple legal patterns we cannot lower to Logical SPIR-V.
10// This pass modifies such loads to have an IR we can directly lower to valid
11// logical SPIR-V.
12// OpenCL can avoid this because they rely on ptrcast, which is not supported
13// by logical SPIR-V.
14//
15// This pass relies on the assign_ptr_type intrinsic to deduce the type of the
16// pointed values, must replace all occurences of `ptrcast`. This is why
17// unhandled cases are reported as unreachable: we MUST cover all cases.
18//
19// 1. Loading the first element of an array
20//
21// %array = [10 x i32]
22// %value = load i32, ptr %array
23//
24// LLVM can skip the GEP instruction, and only request loading the first 4
25// bytes. In logical SPIR-V, we need an OpAccessChain to access the first
26// element. This pass will add a getelementptr instruction before the load.
27//
28//
29// 2. Implicit downcast from load
30//
31// %1 = getelementptr <4 x i32>, ptr %vec4, i64 0
32// %2 = load <3 x i32>, ptr %1
33//
34// The pointer in the GEP instruction is only used for offset computations,
35// but it doesn't NEED to match the pointed type. OpAccessChain however
36// requires this. Also, LLVM loads define the bitwidth of the load, not the
37// pointer. In this example, we can guess %vec4 is a vec4 thanks to the GEP
38// instruction basetype, but we only want to load the first 3 elements, hence
39// do a partial load. In logical SPIR-V, this is not legal. What we must do
40// is load the full vector (basetype), extract 3 elements, and recombine them
41// to form a 3-element vector.
42//
43//===----------------------------------------------------------------------===//
44
45#include "SPIRV.h"
46#include "SPIRVSubtarget.h"
47#include "SPIRVTargetMachine.h"
48#include "SPIRVUtils.h"
49#include "llvm/IR/IRBuilder.h"
51#include "llvm/IR/Intrinsics.h"
52#include "llvm/IR/IntrinsicsSPIRV.h"
55
56using namespace llvm;
57
58namespace {
59class SPIRVLegalizePointerCastImpl {
60
61 // Builds the `spv_assign_type` assigning |Ty| to |Value| at the current
62 // builder position.
63 void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg) {
64 Value *OfType = PoisonValue::get(Ty);
65 CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
66 {Arg->getType()}, OfType, Arg, {}, B);
67 GR->addAssignPtrTypeInstr(Arg, AssignCI);
68 }
69
70 static FixedVectorType *makeVectorFromTotalBits(Type *ElemTy,
71 TypeSize TotalBits) {
72 unsigned ElemBits = ElemTy->getScalarSizeInBits();
73 assert(ElemBits && TotalBits % ElemBits == 0 &&
74 "TotalBits must be divisible by element bit size");
75 return FixedVectorType::get(ElemTy, TotalBits / ElemBits);
76 }
77
78 Value *resizeVectorBitsWithShuffle(IRBuilder<> &B, Value *V,
79 FixedVectorType *DstTy) {
80 auto *SrcTy = cast<FixedVectorType>(V->getType());
81 assert(SrcTy->getElementType() == DstTy->getElementType() &&
82 "shuffle resize expects identical element types");
83
84 const unsigned NumNeeded = DstTy->getNumElements();
85 const unsigned NumSource = SrcTy->getNumElements();
86
87 SmallVector<int> Mask(NumNeeded);
88 for (unsigned I = 0; I < NumNeeded; ++I)
89 Mask[I] = (I < NumSource) ? static_cast<int>(I) : -1;
90
91 Value *Resized = B.CreateShuffleVector(V, V, Mask);
92 buildAssignType(B, DstTy, Resized);
93 return Resized;
94 }
95
96 // Loads parts of the vector of type |SourceType| from the pointer |Source|
97 // and create a new vector of type |TargetType|. |TargetType| must be a vector
98 // type.
99 // Returns the loaded value.
100 Value *loadVectorFromVector(IRBuilder<> &B, FixedVectorType *SourceType,
101 FixedVectorType *TargetType, Value *Source,
102 Align OriginalAlign) {
103 LoadInst *NewLoad = B.CreateLoad(SourceType, Source);
104 NewLoad->setAlignment(OriginalAlign);
105 buildAssignType(B, SourceType, NewLoad);
106 Value *AssignValue = NewLoad;
107 if (TargetType->getElementType() != SourceType->getElementType()) {
108 const DataLayout &DL = B.getDataLayout();
109 TypeSize TargetTypeSize = DL.getTypeSizeInBits(TargetType);
110 TypeSize SourceTypeSize = DL.getTypeSizeInBits(SourceType);
111
112 Value *BitcastSrcVal = NewLoad;
113 FixedVectorType *BitcastSrcTy =
114 cast<FixedVectorType>(BitcastSrcVal->getType());
115 FixedVectorType *BitcastDstTy = TargetType;
116
117 if (TargetTypeSize != SourceTypeSize) {
118 unsigned TargetElemBits =
119 TargetType->getElementType()->getScalarSizeInBits();
120 if (SourceTypeSize % TargetElemBits == 0) {
121 // No Resize needed. Same total bits as source, but use target element
122 // type.
123 BitcastDstTy = makeVectorFromTotalBits(TargetType->getElementType(),
124 SourceTypeSize);
125 } else {
126 // Resize source to target total bitwidth using source element type.
127 BitcastSrcTy = makeVectorFromTotalBits(SourceType->getElementType(),
128 TargetTypeSize);
129 BitcastSrcVal = resizeVectorBitsWithShuffle(B, NewLoad, BitcastSrcTy);
130 }
131 }
132 AssignValue =
133 B.CreateIntrinsic(Intrinsic::spv_bitcast,
134 {BitcastDstTy, BitcastSrcTy}, {BitcastSrcVal});
135 buildAssignType(B, BitcastDstTy, AssignValue);
136 if (BitcastDstTy == TargetType)
137 return AssignValue;
138 }
139
140 auto *AssignVecTy = cast<FixedVectorType>(AssignValue->getType());
141 const unsigned NumTarget = TargetType->getNumElements();
142 const unsigned NumSource = AssignVecTy->getNumElements();
143
144 // Optimizations may widen a narrow load to cover padding (e.g., loading a
145 // <1 x float> column as <4 x float>). Since extra lanes read trailing
146 // padding, insert only the valid lanes into a poison vector to avoid poison
147 // scalars.
148 if (NumTarget > NumSource) {
149 Value *Result = PoisonValue::get(TargetType);
150 buildAssignType(B, TargetType, Result);
151 for (unsigned I = 0; I < NumSource; ++I) {
152 Value *Scalar = extractScalarFromVector(B, AssignValue, I);
153 Result = makeInsertElement(B, Result, Scalar, I);
154 }
155 return Result;
156 }
157
158 assert(NumTarget < NumSource);
159 SmallVector<int> Mask(/* Size= */ NumTarget);
160 for (unsigned I = 0; I < NumTarget; ++I)
161 Mask[I] = I;
162 Value *Output = B.CreateShuffleVector(AssignValue, AssignValue, Mask);
163 buildAssignType(B, TargetType, Output);
164 return Output;
165 }
166
167 // Returns true if |FromTy| has a memory layout compatible with loading or
168 // storing |ToTy|.
169 bool isCompatibleMemoryLayout(Type *ToTy, Type *FromTy) {
170 if (ToTy == FromTy)
171 return true;
172 auto *SVT = dyn_cast<FixedVectorType>(FromTy);
173 auto *DVT = dyn_cast<FixedVectorType>(ToTy);
174 if (SVT && DVT)
175 return true;
176 if (SVT && (ToTy->isIntegerTy() || ToTy->isFloatingPointTy()) &&
177 ToTy->getPrimitiveSizeInBits() == SVT->getPrimitiveSizeInBits())
178 return true;
179 auto *SAT = dyn_cast<ArrayType>(FromTy);
180 if (SAT && DVT) {
181 if (SAT->getElementType() == DVT->getElementType())
182 return true;
183 if (auto *MAT = dyn_cast<FixedVectorType>(SAT->getElementType()))
184 if (MAT->getElementType() == DVT->getElementType())
185 return true;
186 }
187 return false;
188 }
189
190 // Traverses the aggregate type to find the first sub-type that matches
191 // the TargetElemType's memory layout, optionally emitting a GEP intrinsic.
192 std::optional<std::pair<Value *, Type *>>
193 getPointerToFirstCompatibleType(IRBuilder<> &B, Value *BasePtr,
194 Type *PointerType, Type *TargetElemType,
195 bool IsInBounds) {
196 Type *CurrentTy = GR->findDeducedElementType(BasePtr);
197 assert(CurrentTy && "Could not deduce aggregate type");
198 SmallVector<Value *, 8> Args{/* isInBounds= */ B.getInt1(IsInBounds),
199 BasePtr};
200 Args.push_back(B.getInt32(0)); // Pointer offset
201
202 while (!isCompatibleMemoryLayout(TargetElemType, CurrentTy)) {
203 if (auto *ST = dyn_cast<StructType>(CurrentTy)) {
204 if (ST->getNumElements() == 0)
205 return std::nullopt;
206 CurrentTy = ST->getTypeAtIndex(0u);
207 } else if (auto *AT = dyn_cast<ArrayType>(CurrentTy)) {
208 CurrentTy = AT->getElementType();
209 } else if (auto *VT = dyn_cast<FixedVectorType>(CurrentTy)) {
210 CurrentTy = VT->getElementType();
211 } else {
212 return std::nullopt;
213 }
214 Args.push_back(B.getInt32(0));
215 }
216
217 Value *GEP = BasePtr;
218 if (Args.size() > 3) {
219 std::array<Type *, 2> Types = {PointerType, BasePtr->getType()};
220 GEP = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
221 GR->buildAssignPtr(B, CurrentTy, GEP);
222 }
223
224 return std::make_pair(GEP, CurrentTy);
225 }
226
227 static IntrinsicInst *getResourceGetPointer(Value *Ptr) {
228 if (auto *II = dyn_cast<IntrinsicInst>(Ptr))
229 if (II->getIntrinsicID() == Intrinsic::spv_resource_getpointer)
230 return II;
231 return nullptr;
232 }
233
234 Value *gepByteOffset(IRBuilder<> &B, Value *BasePtr, unsigned ByteOffset) {
235 if (ByteOffset == 0)
236 return BasePtr;
237
238 if (IntrinsicInst *ResourcePtr = getResourceGetPointer(BasePtr)) {
239 Value *Handle = ResourcePtr->getOperand(0);
240 Value *BaseOffset = ResourcePtr->getOperand(1);
241 Value *NewOffset;
242 if (auto *CI = dyn_cast<ConstantInt>(BaseOffset)) {
243 NewOffset =
244 ConstantInt::get(CI->getType(), CI->getZExtValue() + ByteOffset);
245 } else {
246 NewOffset = B.CreateAdd(
247 BaseOffset, ConstantInt::get(BaseOffset->getType(), ByteOffset));
248 buildAssignType(B, NewOffset->getType(), NewOffset);
249 }
251 ResourcePtr->getOperandBundlesAsDefs(OpBundles);
252 CallInst *ResourcePtrAtOffset = B.CreateCall(
253 ResourcePtr->getFunctionType(), ResourcePtr->getCalledOperand(),
254 {Handle, NewOffset}, OpBundles);
255 ResourcePtrAtOffset->setAttributes(ResourcePtr->getAttributes());
256 ResourcePtrAtOffset->setCallingConv(ResourcePtr->getCallingConv());
257 Type *I8Ty = Type::getInt8Ty(B.getContext());
258 GR->buildAssignPtr(B, I8Ty, ResourcePtrAtOffset);
259 return ResourcePtrAtOffset;
260 }
262 "byte layout pointer must come from spv.resource.getpointer");
263 }
264
265 Value *scalarToStoreInt(IRBuilder<> &B, Value *Scalar) {
266 Type *Ty = Scalar->getType();
267 const DataLayout &DL = B.getDataLayout();
268 Type *IntTy =
269 IntegerType::get(B.getContext(), DL.getTypeStoreSizeInBits(Ty));
270 if (Ty == IntTy)
271 return Scalar;
272 if (Ty->isIntOrIntVectorTy())
273 return B.CreateIntCast(Scalar, IntTy, /*isSigned=*/false);
274 return B.CreateIntrinsic(Intrinsic::spv_bitcast, {IntTy, Ty}, {Scalar});
275 }
276
277 Value *storeIntToScalar(IRBuilder<> &B, Value *IntVal, Type *ScalarTy) {
278 if (IntVal->getType() == ScalarTy)
279 return IntVal;
280 if (ScalarTy->isIntOrIntVectorTy())
281 return B.CreateIntCast(IntVal, ScalarTy, /*isSigned=*/false);
282 return B.CreateIntrinsic(Intrinsic::spv_bitcast,
283 {ScalarTy, IntVal->getType()}, {IntVal});
284 }
285
286 void storeScalarToByteLayout(IRBuilder<> &B, Value *Src, Value *Dst,
287 Align Alignment) {
288 LLVMContext &Ctx = B.getContext();
289 Type *I8Ty = Type::getInt8Ty(Ctx);
290 const DataLayout &DL = B.getDataLayout();
291 Value *IntVal = scalarToStoreInt(B, Src);
292 if (IntVal != Src)
293 buildAssignType(B, IntVal->getType(), IntVal);
294 unsigned NumBytes = DL.getTypeStoreSize(Src->getType());
295
296 auto StoreByte = [&](unsigned I, Value *Shifted) {
297 Value *Byte = B.CreateTrunc(Shifted, I8Ty);
298 buildAssignType(B, I8Ty, Byte);
299 Value *Ptr = gepByteOffset(B, Dst, I);
300 StoreInst *SI = B.CreateStore(Byte, Ptr);
301 SI->setAlignment(commonAlignment(Alignment, I));
302 };
303
304 if (NumBytes > 0)
305 StoreByte(0, IntVal);
306
307 for (unsigned I = 1; I < NumBytes; ++I) {
308 Value *Shifted =
309 B.CreateLShr(IntVal, ConstantInt::get(IntVal->getType(), 8 * I));
310 StoreByte(I, Shifted);
311 }
312 }
313
314 Value *loadScalarFromByteLayout(IRBuilder<> &B, Type *AccessTy, Value *Src,
315 Align Alignment) {
316 LLVMContext &Ctx = B.getContext();
317 Type *I8Ty = Type::getInt8Ty(Ctx);
318 const DataLayout &DL = B.getDataLayout();
319 unsigned NumBytes = DL.getTypeStoreSize(AccessTy);
320 Type *IntTy = IntegerType::get(Ctx, DL.getTypeStoreSizeInBits(AccessTy));
321 Value *IntVal = ConstantInt::get(IntTy, 0);
322
323 for (unsigned I = 0; I < NumBytes; ++I) {
324 Value *Ptr = gepByteOffset(B, Src, I);
325 LoadInst *LI = B.CreateLoad(I8Ty, Ptr);
326 LI->setAlignment(commonAlignment(Alignment, I));
327 buildAssignType(B, I8Ty, LI);
328 Value *Extended = B.CreateZExt(LI, IntTy);
329 buildAssignType(B, IntTy, Extended);
330
331 if (I == 0) {
333 } else {
334 Value *Shifted = B.CreateShl(Extended, ConstantInt::get(IntTy, 8 * I));
335 buildAssignType(B, IntTy, Shifted);
336 IntVal = B.CreateOr(IntVal, Shifted);
337 }
338 buildAssignType(B, IntTy, IntVal);
339 }
340
341 Value *Result = storeIntToScalar(B, IntVal, AccessTy);
342 if (Result != IntVal)
343 buildAssignType(B, AccessTy, Result);
344 return Result;
345 }
346
347 // Classifies a ptrcast reinterpretation: casted pointee matches the access
348 // type but differs from the original storage layout (e.g. i8 byte buffer as
349 // i32). ByteWise means multi-byte access must use per-byte i8 load/store.
350 bool shouldReinterpretByteWise(IRBuilder<> &B, Type *AccessTy,
351 Value *OriginalPtr) {
352 Type *OriginalElemTy = GR->findDeducedElementType(OriginalPtr);
353 const DataLayout &DL = B.getDataLayout();
354 if (getResourceGetPointer(OriginalPtr) &&
355 OriginalElemTy == Type::getInt8Ty(B.getContext()) &&
356 AccessTy->isSingleValueType() && DL.getTypeStoreSize(AccessTy) > 1)
357 return true;
358
359 return false;
360 }
361
362 static bool canReinterpretType(Type *FromTy, Type *ToTy) {
363 return FromTy && ToTy && CastInst::isBitCastable(FromTy, ToTy);
364 }
365
366 bool tryReinterpretLoad(IRBuilder<> &B, Type *AccessTy, Value *OriginalPtr,
367 Value *CastedPtr, LoadInst *IllegalLoad) {
368 Type *CastedElemTy = GR->findDeducedElementType(CastedPtr);
369 if (!CastedElemTy || CastedElemTy != AccessTy)
370 return false;
371
372 Align Alignment = IllegalLoad->getAlign();
373 if (shouldReinterpretByteWise(B, AccessTy, OriginalPtr)) {
374 const DataLayout &DL = B.getDataLayout();
375 Value *Loaded;
376 if (auto *VT = dyn_cast<FixedVectorType>(AccessTy)) {
377 unsigned ElemSize = DL.getTypeStoreSize(VT->getElementType());
378 SmallVector<Value *, 4> LoadedElements;
379 for (unsigned I = 0; I < VT->getNumElements(); ++I) {
380 Value *ElemPtr = gepByteOffset(B, OriginalPtr, I * ElemSize);
381 LoadedElements.push_back(loadScalarFromByteLayout(
382 B, VT->getElementType(), ElemPtr,
383 commonAlignment(Alignment, I * ElemSize)));
384 }
385 Loaded = buildVectorFromLoadedElements(B, VT, LoadedElements);
386 } else {
387 Loaded = loadScalarFromByteLayout(B, AccessTy, OriginalPtr, Alignment);
388 buildAssignType(B, AccessTy, Loaded);
389 }
390 GR->replaceAllUsesWith(IllegalLoad, Loaded, /* DeleteOld= */ true);
391 DeadInstructions.push_back(IllegalLoad);
392 return true;
393 }
394
395 Type *OriginalTy = GR->findDeducedElementType(OriginalPtr);
396 if (!canReinterpretType(OriginalTy, AccessTy))
398 "Cannot lower a pointer cast without equal-width numeric types for "
399 "Logical SPIR-V",
400 false);
401 LoadInst *LI = B.CreateLoad(OriginalTy, OriginalPtr);
402 LI->setAlignment(Alignment);
403 LI->setVolatile(IllegalLoad->isVolatile());
404 LI->setAtomic(IllegalLoad->getOrdering(), IllegalLoad->getSyncScopeID());
405 buildAssignType(B, OriginalTy, LI);
406 Value *Loaded =
407 B.CreateIntrinsic(Intrinsic::spv_bitcast, {AccessTy, OriginalTy}, {LI});
408 buildAssignType(B, AccessTy, Loaded);
409 GR->replaceAllUsesWith(IllegalLoad, Loaded, /* DeleteOld= */ true);
410 DeadInstructions.push_back(IllegalLoad);
411 return true;
412 }
413
414 bool tryReinterpretStore(IRBuilder<> &B, Type *AccessTy, Value *OriginalPtr,
415 Value *CastedPtr, Value *StoreSrc, Align Alignment,
416 Instruction *IllegalStore) {
417 Type *CastedElemTy = GR->findDeducedElementType(CastedPtr);
418 if (!CastedElemTy || CastedElemTy != AccessTy)
419 return false;
420
421 if (shouldReinterpretByteWise(B, AccessTy, OriginalPtr)) {
422 const DataLayout &DL = B.getDataLayout();
423 if (auto *VT = dyn_cast<FixedVectorType>(StoreSrc->getType())) {
424 unsigned ElemSize = DL.getTypeStoreSize(VT->getElementType());
425 for (unsigned I = 0; I < VT->getNumElements(); ++I) {
426 Value *Elem = extractScalarFromVector(B, StoreSrc, I);
427 Value *ElemPtr = gepByteOffset(B, OriginalPtr, I * ElemSize);
428 storeScalarToByteLayout(B, Elem, ElemPtr,
429 commonAlignment(Alignment, I * ElemSize));
430 }
431 } else {
432 storeScalarToByteLayout(B, StoreSrc, OriginalPtr, Alignment);
433 }
434 return true;
435 }
436
437 Type *OriginalTy = GR->findDeducedElementType(OriginalPtr);
438 if (!canReinterpretType(AccessTy, OriginalTy))
440 "Cannot lower a pointer cast without equal-width numeric types for "
441 "Logical SPIR-V",
442 false);
443 Value *Stored = B.CreateIntrinsic(Intrinsic::spv_bitcast,
444 {OriginalTy, AccessTy}, {StoreSrc});
445 buildAssignType(B, OriginalTy, Stored);
446 StoreInst *SI = B.CreateStore(Stored, OriginalPtr);
447 SI->setAlignment(Alignment);
448 if (auto *OldSI = dyn_cast<StoreInst>(IllegalStore)) {
449 SI->setVolatile(OldSI->isVolatile());
450 SI->setAtomic(OldSI->getOrdering(), OldSI->getSyncScopeID());
451 }
452 return true;
453 }
454
455 // Builds a legalized load from a pointer, drilling down through
456 // memory layouts to find a compatible type. Load flags will be
457 // copied from |IllegalLoad|, which should be the load being legalized.
458 Value *buildLegalizedLoad(IRBuilder<> &B, Type *ElementType, Value *Source,
459 LoadInst *IllegalLoad, Value *CastedPtr) {
460 auto ResultOpt = getPointerToFirstCompatibleType(
461 B, Source, IllegalLoad->getPointerOperandType(), ElementType, false);
462 if (!ResultOpt) {
463 if (tryReinterpretLoad(B, ElementType, Source, CastedPtr, IllegalLoad))
464 return nullptr;
465 llvm_unreachable("Failed to load from aggregate: "
466 "Could not find compatible memory layout.");
467 }
468 auto [GEP, CurrentTy] = *ResultOpt;
469
470 auto *SAT = dyn_cast<ArrayType>(CurrentTy);
471 auto *SVT = dyn_cast<FixedVectorType>(CurrentTy);
472 auto *DVT = dyn_cast<FixedVectorType>(ElementType);
473 auto *MAT =
474 SAT ? dyn_cast<FixedVectorType>(SAT->getElementType()) : nullptr;
475
476 if (ElementType == CurrentTy) {
477 LoadInst *LI = B.CreateLoad(ElementType, GEP);
478 LI->setAlignment(IllegalLoad->getAlign());
479 buildAssignType(B, ElementType, LI);
480 return LI;
481 }
482 if (SVT && !DVT) {
483 LoadInst *LI = B.CreateLoad(CurrentTy, GEP);
484 LI->setAlignment(IllegalLoad->getAlign());
485 buildAssignType(B, CurrentTy, LI);
486 Value *Result = B.CreateIntrinsic(Intrinsic::spv_bitcast,
487 {ElementType, CurrentTy}, {LI});
488 buildAssignType(B, ElementType, Result);
489 return Result;
490 }
491 if (SVT && DVT)
492 return loadVectorFromVector(B, SVT, DVT, GEP, IllegalLoad->getAlign());
493 if (SAT && DVT && SAT->getElementType() == DVT->getElementType())
494 return loadVectorFromArray(B, DVT, GEP, IllegalLoad->getAlign());
495 if (MAT && DVT && MAT->getElementType() == DVT->getElementType())
496 return loadVectorFromMatrixArray(B, DVT, GEP, MAT,
497 IllegalLoad->getAlign());
498
499 llvm_unreachable("Failed to load from aggregate.");
500 }
501
502 Value *
503 buildVectorFromLoadedElements(IRBuilder<> &B, FixedVectorType *TargetType,
504 SmallVector<Value *, 4> &LoadedElements) {
505 // <1 x T> shares the SPIR-V type with T, so emitting OpCompositeInsert on
506 // a scalar would be invalid. Bridge with spv_bitcast instead unless
507 // SPV_EXT_long_vector is available.
508 bool CanUseAnyVectorRank = TM.getSubtargetImpl()->canUseExtension(
509 SPIRV::Extension::SPV_EXT_long_vector);
510 if (TargetType->getNumElements() == 1 && !CanUseAnyVectorRank) {
511 Value *Scalar = LoadedElements[0];
512 Value *NewVector = B.CreateIntrinsic(
513 Intrinsic::spv_bitcast, {TargetType, Scalar->getType()}, {Scalar});
514 buildAssignType(B, TargetType, NewVector);
515 return NewVector;
516 }
517
518 // Build the vector from the loaded elements.
519 Value *NewVector = PoisonValue::get(TargetType);
520 buildAssignType(B, TargetType, NewVector);
521
522 for (unsigned I = 0, E = TargetType->getNumElements(); I < E; ++I) {
523 Value *Index = B.getInt32(I);
525 TargetType->getElementType(),
526 Index->getType()};
527 SmallVector<Value *> Args = {NewVector, LoadedElements[I], Index};
528 NewVector = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
529 buildAssignType(B, TargetType, NewVector);
530 }
531 return NewVector;
532 }
533
534 // Loads elements from a matrix with an array of vector memory layout and
535 // constructs a vector.
536 Value *loadVectorFromMatrixArray(IRBuilder<> &B, FixedVectorType *TargetType,
537 Value *Source, FixedVectorType *ArrElemVecTy,
538 Align OriginalAlign) {
539 Type *TargetElemTy = TargetType->getElementType();
540 unsigned ScalarsPerArrayElement = ArrElemVecTy->getNumElements();
541 const DataLayout &DL = B.getDataLayout();
542 uint64_t ArrElemVecSize = DL.getTypeAllocSize(ArrElemVecTy);
543 // Load each element of the array.
544 SmallVector<Value *, 4> LoadedElements;
545 std::array<Type *, 2> Types = {Source->getType(), Source->getType()};
546 for (unsigned I = 0, E = TargetType->getNumElements(); I < E; ++I) {
547 unsigned ArrayIndex = I / ScalarsPerArrayElement;
548 unsigned ElementIndexInArrayElem = I % ScalarsPerArrayElement;
549 // Create a GEP to access the i-th element of the array.
550 std::array<Value *, 4> Args = {
551 B.getInt1(/*Inbounds=*/false), Source, B.getInt32(0),
552 ConstantInt::get(B.getInt32Ty(), ArrayIndex)};
553 auto *ElementPtr = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
554 GR->buildAssignPtr(B, ArrElemVecTy, ElementPtr);
555 LoadInst *LoadVec = B.CreateLoad(ArrElemVecTy, ElementPtr);
556 LoadVec->setAlignment(
557 commonAlignment(OriginalAlign, ArrayIndex * ArrElemVecSize));
558 buildAssignType(B, ArrElemVecTy, LoadVec);
559 LoadedElements.push_back(makeExtractElement(B, TargetElemTy, LoadVec,
560 ElementIndexInArrayElem));
561 }
562 return buildVectorFromLoadedElements(B, TargetType, LoadedElements);
563 }
564
565 // Loads elements from an array and constructs a vector.
566 Value *loadVectorFromArray(IRBuilder<> &B, FixedVectorType *TargetType,
567 Value *Source, Align OriginalAlign) {
568 // Load each element of the array.
569 SmallVector<Value *, 4> LoadedElements;
570 std::array<Type *, 2> Types = {Source->getType(), Source->getType()};
571 const DataLayout &DL = B.getDataLayout();
572 uint64_t ElemSize = DL.getTypeAllocSize(TargetType->getElementType());
573 for (unsigned I = 0, E = TargetType->getNumElements(); I < E; ++I) {
574 // Create a GEP to access the i-th element of the array.
575 std::array<Value *, 4> Args = {B.getInt1(/*Inbounds=*/false), Source,
576 B.getInt32(0),
577 ConstantInt::get(B.getInt32Ty(), I)};
578 auto *ElementPtr = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
579 GR->buildAssignPtr(B, TargetType->getElementType(), ElementPtr);
580
581 // Load the value from the element pointer.
582 LoadInst *Load = B.CreateLoad(TargetType->getElementType(), ElementPtr);
583 Load->setAlignment(commonAlignment(OriginalAlign, I * ElemSize));
584 buildAssignType(B, TargetType->getElementType(), Load);
585 LoadedElements.push_back(Load);
586 }
587 return buildVectorFromLoadedElements(B, TargetType, LoadedElements);
588 }
589
590 // Stores elements from a vector into a matrix (an array of vectors).
591 void storeMatrixArrayFromVector(IRBuilder<> &B, Value *SrcVector,
592 Value *DstArrayPtr, ArrayType *ArrTy,
593 Align Alignment) {
594 auto *SrcVecTy = cast<FixedVectorType>(SrcVector->getType());
595 auto *ArrElemVecTy = cast<FixedVectorType>(ArrTy->getElementType());
596 Type *ElemTy = ArrElemVecTy->getElementType();
597 unsigned ScalarsPerArrayElement = ArrElemVecTy->getNumElements();
598 unsigned SrcNumElements = SrcVecTy->getNumElements();
599 assert(
600 SrcNumElements % ScalarsPerArrayElement == 0 &&
601 "Source vector size must be a multiple of array element vector size");
602
603 std::array<Type *, 2> Types = {DstArrayPtr->getType(),
604 DstArrayPtr->getType()};
605 const DataLayout &DL = B.getDataLayout();
606 uint64_t ArrElemVecSize = DL.getTypeAllocSize(ArrElemVecTy);
607
608 for (unsigned I = 0; I < SrcNumElements; I += ScalarsPerArrayElement) {
609 unsigned ArrayIndex = I / ScalarsPerArrayElement;
610 // Create a GEP to access the array element.
611 std::array<Value *, 4> Args = {
612 B.getInt1(/*Inbounds=*/false), DstArrayPtr, B.getInt32(0),
613 ConstantInt::get(B.getInt32Ty(), ArrayIndex)};
614 auto *ElementPtr = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
615 GR->buildAssignPtr(B, ArrElemVecTy, ElementPtr);
616
617 // Extract scalar elements from the source vector for this array slot.
618 SmallVector<Value *, 4> Elements;
619 for (unsigned J = 0; J < ScalarsPerArrayElement; ++J)
620 Elements.push_back(makeExtractElement(B, ElemTy, SrcVector, I + J));
621
622 // Build a vector from the extracted elements and store it.
623 Value *Vec = buildVectorFromLoadedElements(B, ArrElemVecTy, Elements);
624 StoreInst *SI = B.CreateStore(Vec, ElementPtr);
625 SI->setAlignment(commonAlignment(Alignment, ArrayIndex * ArrElemVecSize));
626 }
627 }
628
629 // Stores elements from a vector into an array.
630 void storeArrayFromVector(IRBuilder<> &B, Value *SrcVector,
631 Value *DstArrayPtr, ArrayType *ArrTy,
632 Align Alignment) {
633 auto *VecTy = cast<FixedVectorType>(SrcVector->getType());
634 Type *ElemTy = ArrTy->getElementType();
635
636 // Ensure the element types of the array and vector are the same.
637 assert(VecTy->getElementType() == ElemTy &&
638 "Element types of array and vector must be the same.");
639 std::array<Type *, 2> Types = {DstArrayPtr->getType(),
640 DstArrayPtr->getType()};
641 const DataLayout &DL = B.getDataLayout();
642 uint64_t ElemSize = DL.getTypeAllocSize(ElemTy);
643
644 for (unsigned I = 0, E = VecTy->getNumElements(); I < E; ++I) {
645 // Create a GEP to access the i-th element of the array.
646 std::array<Value *, 4> Args = {B.getInt1(/*Inbounds=*/false), DstArrayPtr,
647 B.getInt32(0),
648 ConstantInt::get(B.getInt32Ty(), I)};
649 auto *ElementPtr = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
650 GR->buildAssignPtr(B, ElemTy, ElementPtr);
651
652 // Extract the element from the vector and store it.
653 bool CanUseAnyVectorRank = TM.getSubtargetImpl()->canUseExtension(
654 SPIRV::Extension::SPV_EXT_long_vector);
655 Value *Element = (E == 1 && !CanUseAnyVectorRank)
656 ? SrcVector
657 : makeExtractElement(B, ElemTy, SrcVector, I);
658 StoreInst *SI = B.CreateStore(Element, ElementPtr);
659 SI->setAlignment(commonAlignment(Alignment, I * ElemSize));
660 }
661 }
662
663 // Replaces the load instruction to get rid of the ptrcast used as source
664 // operand.
665 void transformLoad(IRBuilder<> &B, LoadInst *LI, Value *CastedOperand,
666 Value *OriginalOperand) {
667 Type *ToTy = GR->findDeducedElementType(CastedOperand);
668 B.SetInsertPoint(LI);
669
670 Value *Output =
671 buildLegalizedLoad(B, ToTy, OriginalOperand, LI, CastedOperand);
672 if (!Output)
673 return;
674
675 GR->replaceAllUsesWith(LI, Output, /* DeleteOld= */ true);
676 DeadInstructions.push_back(LI);
677 }
678
679 // Creates an spv_insertelt instruction (equivalent to llvm's insertelement).
680 Value *makeInsertElement(IRBuilder<> &B, Value *Vector, Value *Element,
681 unsigned Index) {
682 Type *Int32Ty = Type::getInt32Ty(B.getContext());
683 SmallVector<Type *, 4> Types = {Vector->getType(), Vector->getType(),
684 Element->getType(), Int32Ty};
685 SmallVector<Value *> Args = {Vector, Element, B.getInt32(Index)};
686 Value *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
687 buildAssignType(B, Vector->getType(), NewI);
688 return NewI;
689 }
690
691 // Creates an spv_extractelt instruction (equivalent to llvm's
692 // extractelement).
693 Value *makeExtractElement(IRBuilder<> &B, Type *ElementType, Value *Vector,
694 unsigned Index) {
695 Type *Int32Ty = Type::getInt32Ty(B.getContext());
696 SmallVector<Type *, 3> Types = {ElementType, Vector->getType(), Int32Ty};
697 SmallVector<Value *> Args = {Vector, B.getInt32(Index)};
698 Value *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
699 buildAssignType(B, ElementType, NewI);
700 return NewI;
701 }
702
703 // Extracts scalar element |Index| from |Vector|. A <1 x T> vector shares its
704 // SPIR-V type with the scalar T, so a plain extractelement would be invalid;
705 // bridge it with spv_bitcast instead.
706 Value *extractScalarFromVector(IRBuilder<> &B, Value *Vector,
707 unsigned Index) {
708 auto *VecTy = cast<FixedVectorType>(Vector->getType());
709 Type *ElemTy = VecTy->getElementType();
710 if (VecTy->getNumElements() == 1) {
711 Value *Scalar =
712 B.CreateIntrinsic(Intrinsic::spv_bitcast, {ElemTy, VecTy}, {Vector});
713 buildAssignType(B, ElemTy, Scalar);
714 return Scalar;
715 }
716 return makeExtractElement(B, ElemTy, Vector, Index);
717 }
718
719 // Stores the given Src vector operand into the Dst vector, adjusting the size
720 // if required.
721 Value *storeVectorFromVector(IRBuilder<> &B, Value *Src, Value *Dst,
722 Align Alignment) {
723 FixedVectorType *SrcType = cast<FixedVectorType>(Src->getType());
724 FixedVectorType *DstType =
725 cast<FixedVectorType>(GR->findDeducedElementType(Dst));
726 auto dstNumElements = DstType->getNumElements();
727 auto srcNumElements = SrcType->getNumElements();
728
729 // if the element type differs, it is a bitcast.
730 if (DstType->getElementType() != SrcType->getElementType()) {
731 // Support bitcast between vectors of different sizes only if
732 // the total bitwidth is the same.
733 [[maybe_unused]] auto dstBitWidth =
734 DstType->getElementType()->getScalarSizeInBits() * dstNumElements;
735 [[maybe_unused]] auto srcBitWidth =
736 SrcType->getElementType()->getScalarSizeInBits() * srcNumElements;
737 assert(dstBitWidth == srcBitWidth &&
738 "Unsupported bitcast between vectors of different sizes.");
739
740 Src =
741 B.CreateIntrinsic(Intrinsic::spv_bitcast, {DstType, SrcType}, {Src});
742 buildAssignType(B, DstType, Src);
743
744 StoreInst *SI = B.CreateStore(Src, Dst);
745 SI->setAlignment(Alignment);
746 return SI;
747 }
748
749 assert(DstType->getNumElements() >= SrcType->getNumElements());
750 LoadInst *LI = B.CreateLoad(DstType, Dst);
751 LI->setAlignment(Alignment);
752 Value *OldValues = LI;
753 buildAssignType(B, OldValues->getType(), OldValues);
754
755 for (unsigned I = 0; I < SrcType->getNumElements(); ++I) {
756 Value *Element = extractScalarFromVector(B, Src, I);
757 OldValues = makeInsertElement(B, OldValues, Element, I);
758 }
759
760 StoreInst *SI = B.CreateStore(OldValues, Dst);
761 SI->setAlignment(Alignment);
762 return SI;
763 }
764
765 // Builds a legalized store to a pointer, drilling down through
766 // memory layouts to find a compatible type.
767 void buildLegalizedStore(IRBuilder<> &B, Value *Src, Value *Dst,
768 Align Alignment, Value *CastedPtr,
769 Instruction *IllegalStore) {
770 auto ResultOpt = getPointerToFirstCompatibleType(B, Dst, Dst->getType(),
771 Src->getType(), true);
772 if (!ResultOpt) {
773 if (tryReinterpretStore(B, Src->getType(), Dst, CastedPtr, Src, Alignment,
774 IllegalStore))
775 return;
776 llvm_unreachable("Failed to store to aggregate: "
777 "Could not find compatible memory layout.");
778 }
779 auto [GEP, CurrentTy] = *ResultOpt;
780
781 auto *DAT = dyn_cast<ArrayType>(CurrentTy);
782 auto *DVT = dyn_cast<FixedVectorType>(CurrentTy);
783 auto *SVT = dyn_cast<FixedVectorType>(Src->getType());
784 auto *DMAT =
785 DAT ? dyn_cast<FixedVectorType>(DAT->getElementType()) : nullptr;
786
787 if (Src->getType() == CurrentTy) {
788 StoreInst *SI = B.CreateStore(Src, GEP);
789 SI->setAlignment(Alignment);
790 return;
791 }
792 if (DVT && !SVT) {
793 Value *Result = B.CreateIntrinsic(Intrinsic::spv_bitcast,
794 {CurrentTy, Src->getType()}, {Src});
795 buildAssignType(B, CurrentTy, Result);
796 StoreInst *SI = B.CreateStore(Result, GEP);
797 SI->setAlignment(Alignment);
798 return;
799 }
800 if (DVT && SVT) {
801 storeVectorFromVector(B, Src, GEP, Alignment);
802 return;
803 }
804 if (DAT && SVT && SVT->getElementType() == DAT->getElementType()) {
805 storeArrayFromVector(B, Src, GEP, DAT, Alignment);
806 return;
807 }
808 if (DMAT && SVT && DMAT->getElementType() == SVT->getElementType()) {
809 storeMatrixArrayFromVector(B, Src, GEP, DAT, Alignment);
810 return;
811 }
812
813 llvm_unreachable("Failed to store to aggregate.");
814 }
815
816 // Transforms a store instruction (or SPV intrinsic) using a ptrcast as
817 // operand into a valid logical SPIR-V store with no ptrcast.
818 void transformStore(IRBuilder<> &B, Instruction *IllegalStore, Value *Src,
819 Value *Dst, Value *CastedOperand, Align Alignment) {
820 B.SetInsertPoint(IllegalStore);
821 buildLegalizedStore(B, Src, Dst, Alignment, CastedOperand, IllegalStore);
822 DeadInstructions.push_back(IllegalStore);
823 }
824
825 void legalizePointerCast(IntrinsicInst *II) {
826 Value *CastedOperand = II;
827 Value *OriginalOperand = II->getOperand(0);
828
829 IRBuilder<> B(*II->getModule());
830 std::vector<Value *> Users;
831 for (Use &U : II->uses())
832 Users.push_back(U.getUser());
833
834 for (Value *User : Users) {
835 if (LoadInst *LI = dyn_cast<LoadInst>(User)) {
836 transformLoad(B, LI, CastedOperand, OriginalOperand);
837 continue;
838 }
839
840 if (StoreInst *SI = dyn_cast<StoreInst>(User)) {
841 transformStore(B, SI, SI->getValueOperand(), OriginalOperand,
842 CastedOperand, SI->getAlign());
843 continue;
844 }
845
846 if (IntrinsicInst *Intrin = dyn_cast<IntrinsicInst>(User)) {
847 if (Intrin->getIntrinsicID() == Intrinsic::spv_assign_ptr_type) {
848 DeadInstructions.push_back(Intrin);
849 continue;
850 }
851
852 if (Intrin->getIntrinsicID() == Intrinsic::spv_gep) {
853 GR->replaceAllUsesWith(CastedOperand, OriginalOperand,
854 /* DeleteOld= */ false);
855 continue;
856 }
857
858 if (Intrin->getIntrinsicID() == Intrinsic::spv_store) {
860 if (ConstantInt *C = dyn_cast<ConstantInt>(Intrin->getOperand(3)))
861 Alignment = Align(C->getZExtValue());
862 transformStore(B, Intrin, Intrin->getArgOperand(0), OriginalOperand,
863 CastedOperand, Alignment);
864 continue;
865 }
866 }
867
868 llvm_unreachable("Unsupported ptrcast user. Please fix.");
869 }
870
871 DeadInstructions.push_back(II);
872 }
873
874public:
875 SPIRVLegalizePointerCastImpl(const SPIRVTargetMachine &TM) : TM(TM) {}
876
877 bool run(Function &F) {
878 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
879 GR = ST.getSPIRVGlobalRegistry();
880 DeadInstructions.clear();
881
882 std::vector<IntrinsicInst *> WorkList;
883 for (auto &BB : F) {
884 for (auto &I : BB) {
885 auto *II = dyn_cast<IntrinsicInst>(&I);
886 if (II && II->getIntrinsicID() == Intrinsic::spv_ptrcast)
887 WorkList.push_back(II);
888 }
889 }
890
891 for (IntrinsicInst *II : WorkList)
892 legalizePointerCast(II);
893
894 for (Instruction *I : DeadInstructions)
895 I->eraseFromParent();
896
897 return DeadInstructions.size() != 0;
898 }
899
900private:
901 const SPIRVTargetMachine &TM;
902 SPIRVGlobalRegistry *GR = nullptr;
903 std::vector<Instruction *> DeadInstructions;
904};
905
906class SPIRVLegalizePointerCastLegacy : public FunctionPass {
907public:
908 static char ID;
909 SPIRVLegalizePointerCastLegacy(const SPIRVTargetMachine &TM)
910 : FunctionPass(ID), TM(TM) {}
911
912 bool runOnFunction(Function &F) override {
913 return SPIRVLegalizePointerCastImpl(TM).run(F);
914 }
915
916private:
917 const SPIRVTargetMachine &TM;
918};
919} // namespace
920
923 return SPIRVLegalizePointerCastImpl(TM).run(F) ? PreservedAnalyses::none()
925}
926
927char SPIRVLegalizePointerCastLegacy::ID = 0;
928INITIALIZE_PASS(SPIRVLegalizePointerCastLegacy, "spirv-legalize-pointer-cast",
929 "SPIRV legalize pointer cast pass", false, false)
930
932 return new SPIRVLegalizePointerCastLegacy(*TM);
933}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool runOnFunction(Function &F, bool PostInlining)
Hexagon Common GEP
iv Induction Variable Users
Definition IVUsers.cpp:48
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
void setCallingConv(CallingConv::ID CC)
void setAttributes(AttributeList A)
Set the attributes for this call.
static LLVM_ABI bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
unsigned getNumElements() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
void setAlignment(Align Align)
bool isVolatile() const
Return true if this is a load from a volatile memory location.
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this load instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Type * getPointerOperandType() const
void setVolatile(bool V)
Specify whether this is a volatile load or not.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Align getAlign() const
Return the alignment of the access that is being performed.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition Analysis.h:115
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
void push_back(const T &Elt)
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition Type.h:258
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
Definition Type.h:306
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
Type * getElementType() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
ElementType
The element type of an SRV or UAV resource.
Definition DXILABI.h:68
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Load
The value being inserted comes from a load (InsertElement only).
CallInst * buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef< Type * > Types, Value *Arg, Value *Arg2, ArrayRef< Constant * > Imms, IRBuilder<> &B)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
FunctionPass * createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM)