File: | build/source/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp |
Warning: | line 882, column 5 Value stored to 'Call' is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This pass abstracted struct/union member accesses in order to support |
10 | // compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program |
11 | // which can run on different kernels. In particular, if bpf program tries to |
12 | // access a particular kernel data structure member, the details of the |
13 | // intermediate member access will be remembered so bpf loader can do |
14 | // necessary adjustment right before program loading. |
15 | // |
16 | // For example, |
17 | // |
18 | // struct s { |
19 | // int a; |
20 | // int b; |
21 | // }; |
22 | // struct t { |
23 | // struct s c; |
24 | // int d; |
25 | // }; |
26 | // struct t e; |
27 | // |
28 | // For the member access e.c.b, the compiler will generate code |
29 | // &e + 4 |
30 | // |
31 | // The compile-once run-everywhere instead generates the following code |
32 | // r = 4 |
33 | // &e + r |
34 | // The "4" in "r = 4" can be changed based on a particular kernel version. |
35 | // For example, on a particular kernel version, if struct s is changed to |
36 | // |
37 | // struct s { |
38 | // int new_field; |
39 | // int a; |
40 | // int b; |
41 | // } |
42 | // |
43 | // By repeating the member access on the host, the bpf loader can |
44 | // adjust "r = 4" as "r = 8". |
45 | // |
46 | // This feature relies on the following three intrinsic calls: |
47 | // addr = preserve_array_access_index(base, dimension, index) |
48 | // addr = preserve_union_access_index(base, di_index) |
49 | // !llvm.preserve.access.index <union_ditype> |
50 | // addr = preserve_struct_access_index(base, gep_index, di_index) |
51 | // !llvm.preserve.access.index <struct_ditype> |
52 | // |
53 | // Bitfield member access needs special attention. User cannot take the |
54 | // address of a bitfield acceess. To facilitate kernel verifier |
55 | // for easy bitfield code optimization, a new clang intrinsic is introduced: |
56 | // uint32_t __builtin_preserve_field_info(member_access, info_kind) |
57 | // In IR, a chain with two (or more) intrinsic calls will be generated: |
58 | // ... |
59 | // addr = preserve_struct_access_index(base, 1, 1) !struct s |
60 | // uint32_t result = bpf_preserve_field_info(addr, info_kind) |
61 | // |
62 | // Suppose the info_kind is FIELD_SIGNEDNESS, |
63 | // The above two IR intrinsics will be replaced with |
64 | // a relocatable insn: |
65 | // signness = /* signness of member_access */ |
66 | // and signness can be changed by bpf loader based on the |
67 | // types on the host. |
68 | // |
69 | // User can also test whether a field exists or not with |
70 | // uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE) |
71 | // The field will be always available (result = 1) during initial |
72 | // compilation, but bpf loader can patch with the correct value |
73 | // on the target host where the member_access may or may not be available |
74 | // |
75 | //===----------------------------------------------------------------------===// |
76 | |
77 | #include "BPF.h" |
78 | #include "BPFCORE.h" |
79 | #include "BPFTargetMachine.h" |
80 | #include "llvm/BinaryFormat/Dwarf.h" |
81 | #include "llvm/IR/DebugInfoMetadata.h" |
82 | #include "llvm/IR/GlobalVariable.h" |
83 | #include "llvm/IR/Instruction.h" |
84 | #include "llvm/IR/Instructions.h" |
85 | #include "llvm/IR/IntrinsicsBPF.h" |
86 | #include "llvm/IR/Module.h" |
87 | #include "llvm/IR/PassManager.h" |
88 | #include "llvm/IR/Type.h" |
89 | #include "llvm/IR/User.h" |
90 | #include "llvm/IR/Value.h" |
91 | #include "llvm/Pass.h" |
92 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
93 | #include <stack> |
94 | |
95 | #define DEBUG_TYPE"bpf-abstract-member-access" "bpf-abstract-member-access" |
96 | |
97 | namespace llvm { |
98 | constexpr StringRef BPFCoreSharedInfo::AmaAttr; |
99 | uint32_t BPFCoreSharedInfo::SeqNum; |
100 | |
101 | Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB, |
102 | Instruction *Input, |
103 | Instruction *Before) { |
104 | Function *Fn = Intrinsic::getDeclaration( |
105 | M, Intrinsic::bpf_passthrough, {Input->getType(), Input->getType()}); |
106 | Constant *SeqNumVal = ConstantInt::get(Type::getInt32Ty(BB->getContext()), |
107 | BPFCoreSharedInfo::SeqNum++); |
108 | |
109 | auto *NewInst = CallInst::Create(Fn, {SeqNumVal, Input}); |
110 | NewInst->insertBefore(Before); |
111 | return NewInst; |
112 | } |
113 | } // namespace llvm |
114 | |
115 | using namespace llvm; |
116 | |
117 | namespace { |
118 | class BPFAbstractMemberAccess final { |
119 | public: |
120 | BPFAbstractMemberAccess(BPFTargetMachine *TM) : TM(TM) {} |
121 | |
122 | bool run(Function &F); |
123 | |
124 | struct CallInfo { |
125 | uint32_t Kind; |
126 | uint32_t AccessIndex; |
127 | MaybeAlign RecordAlignment; |
128 | MDNode *Metadata; |
129 | WeakTrackingVH Base; |
130 | }; |
131 | typedef std::stack<std::pair<CallInst *, CallInfo>> CallInfoStack; |
132 | |
133 | private: |
134 | enum : uint32_t { |
135 | BPFPreserveArrayAI = 1, |
136 | BPFPreserveUnionAI = 2, |
137 | BPFPreserveStructAI = 3, |
138 | BPFPreserveFieldInfoAI = 4, |
139 | }; |
140 | |
141 | TargetMachine *TM; |
142 | const DataLayout *DL = nullptr; |
143 | Module *M = nullptr; |
144 | |
145 | static std::map<std::string, GlobalVariable *> GEPGlobals; |
146 | // A map to link preserve_*_access_index intrinsic calls. |
147 | std::map<CallInst *, std::pair<CallInst *, CallInfo>> AIChain; |
148 | // A map to hold all the base preserve_*_access_index intrinsic calls. |
149 | // The base call is not an input of any other preserve_* |
150 | // intrinsics. |
151 | std::map<CallInst *, CallInfo> BaseAICalls; |
152 | // A map to hold <AnonRecord, TypeDef> relationships |
153 | std::map<DICompositeType *, DIDerivedType *> AnonRecords; |
154 | |
155 | void CheckAnonRecordType(DIDerivedType *ParentTy, DIType *Ty); |
156 | void CheckCompositeType(DIDerivedType *ParentTy, DICompositeType *CTy); |
157 | void CheckDerivedType(DIDerivedType *ParentTy, DIDerivedType *DTy); |
158 | void ResetMetadata(struct CallInfo &CInfo); |
159 | |
160 | bool doTransformation(Function &F); |
161 | |
162 | void traceAICall(CallInst *Call, CallInfo &ParentInfo); |
163 | void traceBitCast(BitCastInst *BitCast, CallInst *Parent, |
164 | CallInfo &ParentInfo); |
165 | void traceGEP(GetElementPtrInst *GEP, CallInst *Parent, |
166 | CallInfo &ParentInfo); |
167 | void collectAICallChains(Function &F); |
168 | |
169 | bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo); |
170 | bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI, |
171 | const MDNode *ChildMeta); |
172 | bool removePreserveAccessIndexIntrinsic(Function &F); |
173 | void replaceWithGEP(std::vector<CallInst *> &CallList, |
174 | uint32_t NumOfZerosIndex, uint32_t DIIndex); |
175 | bool HasPreserveFieldInfoCall(CallInfoStack &CallStack); |
176 | void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment, |
177 | uint32_t &StartBitOffset, uint32_t &EndBitOffset); |
178 | uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy, |
179 | uint32_t AccessIndex, uint32_t PatchImm, |
180 | MaybeAlign RecordAlignment); |
181 | |
182 | Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo, |
183 | std::string &AccessKey, MDNode *&BaseMeta); |
184 | MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo, |
185 | std::string &AccessKey, bool &IsInt32Ret); |
186 | uint64_t getConstant(const Value *IndexValue); |
187 | bool transformGEPChain(CallInst *Call, CallInfo &CInfo); |
188 | }; |
189 | |
190 | std::map<std::string, GlobalVariable *> BPFAbstractMemberAccess::GEPGlobals; |
191 | |
192 | class BPFAbstractMemberAccessLegacyPass final : public FunctionPass { |
193 | BPFTargetMachine *TM; |
194 | |
195 | bool runOnFunction(Function &F) override { |
196 | return BPFAbstractMemberAccess(TM).run(F); |
197 | } |
198 | |
199 | public: |
200 | static char ID; |
201 | |
202 | // Add optional BPFTargetMachine parameter so that BPF backend can add the |
203 | // phase with target machine to find out the endianness. The default |
204 | // constructor (without parameters) is used by the pass manager for managing |
205 | // purposes. |
206 | BPFAbstractMemberAccessLegacyPass(BPFTargetMachine *TM = nullptr) |
207 | : FunctionPass(ID), TM(TM) {} |
208 | }; |
209 | |
210 | } // End anonymous namespace |
211 | |
212 | char BPFAbstractMemberAccessLegacyPass::ID = 0; |
213 | INITIALIZE_PASS(BPFAbstractMemberAccessLegacyPass, DEBUG_TYPE,static void *initializeBPFAbstractMemberAccessLegacyPassPassOnce (PassRegistry &Registry) { PassInfo *PI = new PassInfo( "BPF Abstract Member Access" , "bpf-abstract-member-access", &BPFAbstractMemberAccessLegacyPass ::ID, PassInfo::NormalCtor_t(callDefaultCtor<BPFAbstractMemberAccessLegacyPass >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeBPFAbstractMemberAccessLegacyPassPassFlag ; void llvm::initializeBPFAbstractMemberAccessLegacyPassPass( PassRegistry &Registry) { llvm::call_once(InitializeBPFAbstractMemberAccessLegacyPassPassFlag , initializeBPFAbstractMemberAccessLegacyPassPassOnce, std::ref (Registry)); } |
214 | "BPF Abstract Member Access", false, false)static void *initializeBPFAbstractMemberAccessLegacyPassPassOnce (PassRegistry &Registry) { PassInfo *PI = new PassInfo( "BPF Abstract Member Access" , "bpf-abstract-member-access", &BPFAbstractMemberAccessLegacyPass ::ID, PassInfo::NormalCtor_t(callDefaultCtor<BPFAbstractMemberAccessLegacyPass >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeBPFAbstractMemberAccessLegacyPassPassFlag ; void llvm::initializeBPFAbstractMemberAccessLegacyPassPass( PassRegistry &Registry) { llvm::call_once(InitializeBPFAbstractMemberAccessLegacyPassPassFlag , initializeBPFAbstractMemberAccessLegacyPassPassOnce, std::ref (Registry)); } |
215 | |
216 | FunctionPass *llvm::createBPFAbstractMemberAccess(BPFTargetMachine *TM) { |
217 | return new BPFAbstractMemberAccessLegacyPass(TM); |
218 | } |
219 | |
220 | bool BPFAbstractMemberAccess::run(Function &F) { |
221 | LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-abstract-member-access")) { dbgs() << "********** Abstract Member Accesses **********\n" ; } } while (false); |
222 | |
223 | M = F.getParent(); |
224 | if (!M) |
225 | return false; |
226 | |
227 | // Bail out if no debug info. |
228 | if (M->debug_compile_units().empty()) |
229 | return false; |
230 | |
231 | // For each argument/return/local_variable type, trace the type |
232 | // pattern like '[derived_type]* [composite_type]' to check |
233 | // and remember (anon record -> typedef) relations where the |
234 | // anon record is defined as |
235 | // typedef [const/volatile/restrict]* [anon record] |
236 | DISubprogram *SP = F.getSubprogram(); |
237 | if (SP && SP->isDefinition()) { |
238 | for (DIType *Ty: SP->getType()->getTypeArray()) |
239 | CheckAnonRecordType(nullptr, Ty); |
240 | for (const DINode *DN : SP->getRetainedNodes()) { |
241 | if (const auto *DV = dyn_cast<DILocalVariable>(DN)) |
242 | CheckAnonRecordType(nullptr, DV->getType()); |
243 | } |
244 | } |
245 | |
246 | DL = &M->getDataLayout(); |
247 | return doTransformation(F); |
248 | } |
249 | |
250 | void BPFAbstractMemberAccess::ResetMetadata(struct CallInfo &CInfo) { |
251 | if (auto Ty = dyn_cast<DICompositeType>(CInfo.Metadata)) { |
252 | if (AnonRecords.find(Ty) != AnonRecords.end()) { |
253 | if (AnonRecords[Ty] != nullptr) |
254 | CInfo.Metadata = AnonRecords[Ty]; |
255 | } |
256 | } |
257 | } |
258 | |
259 | void BPFAbstractMemberAccess::CheckCompositeType(DIDerivedType *ParentTy, |
260 | DICompositeType *CTy) { |
261 | if (!CTy->getName().empty() || !ParentTy || |
262 | ParentTy->getTag() != dwarf::DW_TAG_typedef) |
263 | return; |
264 | |
265 | if (AnonRecords.find(CTy) == AnonRecords.end()) { |
266 | AnonRecords[CTy] = ParentTy; |
267 | return; |
268 | } |
269 | |
270 | // Two or more typedef's may point to the same anon record. |
271 | // If this is the case, set the typedef DIType to be nullptr |
272 | // to indicate the duplication case. |
273 | DIDerivedType *CurrTy = AnonRecords[CTy]; |
274 | if (CurrTy == ParentTy) |
275 | return; |
276 | AnonRecords[CTy] = nullptr; |
277 | } |
278 | |
279 | void BPFAbstractMemberAccess::CheckDerivedType(DIDerivedType *ParentTy, |
280 | DIDerivedType *DTy) { |
281 | DIType *BaseType = DTy->getBaseType(); |
282 | if (!BaseType) |
283 | return; |
284 | |
285 | unsigned Tag = DTy->getTag(); |
286 | if (Tag == dwarf::DW_TAG_pointer_type) |
287 | CheckAnonRecordType(nullptr, BaseType); |
288 | else if (Tag == dwarf::DW_TAG_typedef) |
289 | CheckAnonRecordType(DTy, BaseType); |
290 | else |
291 | CheckAnonRecordType(ParentTy, BaseType); |
292 | } |
293 | |
294 | void BPFAbstractMemberAccess::CheckAnonRecordType(DIDerivedType *ParentTy, |
295 | DIType *Ty) { |
296 | if (!Ty) |
297 | return; |
298 | |
299 | if (auto *CTy = dyn_cast<DICompositeType>(Ty)) |
300 | return CheckCompositeType(ParentTy, CTy); |
301 | else if (auto *DTy = dyn_cast<DIDerivedType>(Ty)) |
302 | return CheckDerivedType(ParentTy, DTy); |
303 | } |
304 | |
305 | static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) { |
306 | if (Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type && |
307 | Tag != dwarf::DW_TAG_volatile_type && |
308 | Tag != dwarf::DW_TAG_restrict_type && |
309 | Tag != dwarf::DW_TAG_member) |
310 | return false; |
311 | if (Tag == dwarf::DW_TAG_typedef && !skipTypedef) |
312 | return false; |
313 | return true; |
314 | } |
315 | |
316 | static DIType * stripQualifiers(DIType *Ty, bool skipTypedef = true) { |
317 | while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) { |
318 | if (!SkipDIDerivedTag(DTy->getTag(), skipTypedef)) |
319 | break; |
320 | Ty = DTy->getBaseType(); |
321 | } |
322 | return Ty; |
323 | } |
324 | |
325 | static const DIType * stripQualifiers(const DIType *Ty) { |
326 | while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) { |
327 | if (!SkipDIDerivedTag(DTy->getTag(), true)) |
328 | break; |
329 | Ty = DTy->getBaseType(); |
330 | } |
331 | return Ty; |
332 | } |
333 | |
334 | static uint32_t calcArraySize(const DICompositeType *CTy, uint32_t StartDim) { |
335 | DINodeArray Elements = CTy->getElements(); |
336 | uint32_t DimSize = 1; |
337 | for (uint32_t I = StartDim; I < Elements.size(); ++I) { |
338 | if (auto *Element = dyn_cast_or_null<DINode>(Elements[I])) |
339 | if (Element->getTag() == dwarf::DW_TAG_subrange_type) { |
340 | const DISubrange *SR = cast<DISubrange>(Element); |
341 | auto *CI = SR->getCount().dyn_cast<ConstantInt *>(); |
342 | DimSize *= CI->getSExtValue(); |
343 | } |
344 | } |
345 | |
346 | return DimSize; |
347 | } |
348 | |
349 | static Type *getBaseElementType(const CallInst *Call) { |
350 | // Element type is stored in an elementtype() attribute on the first param. |
351 | return Call->getParamElementType(0); |
352 | } |
353 | |
354 | /// Check whether a call is a preserve_*_access_index intrinsic call or not. |
355 | bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call, |
356 | CallInfo &CInfo) { |
357 | if (!Call) |
358 | return false; |
359 | |
360 | const auto *GV = dyn_cast<GlobalValue>(Call->getCalledOperand()); |
361 | if (!GV) |
362 | return false; |
363 | if (GV->getName().startswith("llvm.preserve.array.access.index")) { |
364 | CInfo.Kind = BPFPreserveArrayAI; |
365 | CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); |
366 | if (!CInfo.Metadata) |
367 | report_fatal_error("Missing metadata for llvm.preserve.array.access.index intrinsic"); |
368 | CInfo.AccessIndex = getConstant(Call->getArgOperand(2)); |
369 | CInfo.Base = Call->getArgOperand(0); |
370 | CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call)); |
371 | return true; |
372 | } |
373 | if (GV->getName().startswith("llvm.preserve.union.access.index")) { |
374 | CInfo.Kind = BPFPreserveUnionAI; |
375 | CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); |
376 | if (!CInfo.Metadata) |
377 | report_fatal_error("Missing metadata for llvm.preserve.union.access.index intrinsic"); |
378 | ResetMetadata(CInfo); |
379 | CInfo.AccessIndex = getConstant(Call->getArgOperand(1)); |
380 | CInfo.Base = Call->getArgOperand(0); |
381 | return true; |
382 | } |
383 | if (GV->getName().startswith("llvm.preserve.struct.access.index")) { |
384 | CInfo.Kind = BPFPreserveStructAI; |
385 | CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); |
386 | if (!CInfo.Metadata) |
387 | report_fatal_error("Missing metadata for llvm.preserve.struct.access.index intrinsic"); |
388 | ResetMetadata(CInfo); |
389 | CInfo.AccessIndex = getConstant(Call->getArgOperand(2)); |
390 | CInfo.Base = Call->getArgOperand(0); |
391 | CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call)); |
392 | return true; |
393 | } |
394 | if (GV->getName().startswith("llvm.bpf.preserve.field.info")) { |
395 | CInfo.Kind = BPFPreserveFieldInfoAI; |
396 | CInfo.Metadata = nullptr; |
397 | // Check validity of info_kind as clang did not check this. |
398 | uint64_t InfoKind = getConstant(Call->getArgOperand(1)); |
399 | if (InfoKind >= BPFCoreSharedInfo::MAX_FIELD_RELOC_KIND) |
400 | report_fatal_error("Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic"); |
401 | CInfo.AccessIndex = InfoKind; |
402 | return true; |
403 | } |
404 | if (GV->getName().startswith("llvm.bpf.preserve.type.info")) { |
405 | CInfo.Kind = BPFPreserveFieldInfoAI; |
406 | CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); |
407 | if (!CInfo.Metadata) |
408 | report_fatal_error("Missing metadata for llvm.preserve.type.info intrinsic"); |
409 | uint64_t Flag = getConstant(Call->getArgOperand(1)); |
410 | if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_TYPE_INFO_FLAG) |
411 | report_fatal_error("Incorrect flag for llvm.bpf.preserve.type.info intrinsic"); |
412 | if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_EXISTENCE) |
413 | CInfo.AccessIndex = BPFCoreSharedInfo::TYPE_EXISTENCE; |
414 | else if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_MATCH) |
415 | CInfo.AccessIndex = BPFCoreSharedInfo::TYPE_MATCH; |
416 | else |
417 | CInfo.AccessIndex = BPFCoreSharedInfo::TYPE_SIZE; |
418 | return true; |
419 | } |
420 | if (GV->getName().startswith("llvm.bpf.preserve.enum.value")) { |
421 | CInfo.Kind = BPFPreserveFieldInfoAI; |
422 | CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); |
423 | if (!CInfo.Metadata) |
424 | report_fatal_error("Missing metadata for llvm.preserve.enum.value intrinsic"); |
425 | uint64_t Flag = getConstant(Call->getArgOperand(2)); |
426 | if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_ENUM_VALUE_FLAG) |
427 | report_fatal_error("Incorrect flag for llvm.bpf.preserve.enum.value intrinsic"); |
428 | if (Flag == BPFCoreSharedInfo::PRESERVE_ENUM_VALUE_EXISTENCE) |
429 | CInfo.AccessIndex = BPFCoreSharedInfo::ENUM_VALUE_EXISTENCE; |
430 | else |
431 | CInfo.AccessIndex = BPFCoreSharedInfo::ENUM_VALUE; |
432 | return true; |
433 | } |
434 | |
435 | return false; |
436 | } |
437 | |
438 | void BPFAbstractMemberAccess::replaceWithGEP(std::vector<CallInst *> &CallList, |
439 | uint32_t DimensionIndex, |
440 | uint32_t GEPIndex) { |
441 | for (auto *Call : CallList) { |
442 | uint32_t Dimension = 1; |
443 | if (DimensionIndex > 0) |
444 | Dimension = getConstant(Call->getArgOperand(DimensionIndex)); |
445 | |
446 | Constant *Zero = |
447 | ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0); |
448 | SmallVector<Value *, 4> IdxList; |
449 | for (unsigned I = 0; I < Dimension; ++I) |
450 | IdxList.push_back(Zero); |
451 | IdxList.push_back(Call->getArgOperand(GEPIndex)); |
452 | |
453 | auto *GEP = GetElementPtrInst::CreateInBounds( |
454 | getBaseElementType(Call), Call->getArgOperand(0), IdxList, "", Call); |
455 | Call->replaceAllUsesWith(GEP); |
456 | Call->eraseFromParent(); |
457 | } |
458 | } |
459 | |
460 | bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) { |
461 | std::vector<CallInst *> PreserveArrayIndexCalls; |
462 | std::vector<CallInst *> PreserveUnionIndexCalls; |
463 | std::vector<CallInst *> PreserveStructIndexCalls; |
464 | bool Found = false; |
465 | |
466 | for (auto &BB : F) |
467 | for (auto &I : BB) { |
468 | auto *Call = dyn_cast<CallInst>(&I); |
469 | CallInfo CInfo; |
470 | if (!IsPreserveDIAccessIndexCall(Call, CInfo)) |
471 | continue; |
472 | |
473 | Found = true; |
474 | if (CInfo.Kind == BPFPreserveArrayAI) |
475 | PreserveArrayIndexCalls.push_back(Call); |
476 | else if (CInfo.Kind == BPFPreserveUnionAI) |
477 | PreserveUnionIndexCalls.push_back(Call); |
478 | else |
479 | PreserveStructIndexCalls.push_back(Call); |
480 | } |
481 | |
482 | // do the following transformation: |
483 | // . addr = preserve_array_access_index(base, dimension, index) |
484 | // is transformed to |
485 | // addr = GEP(base, dimenion's zero's, index) |
486 | // . addr = preserve_union_access_index(base, di_index) |
487 | // is transformed to |
488 | // addr = base, i.e., all usages of "addr" are replaced by "base". |
489 | // . addr = preserve_struct_access_index(base, gep_index, di_index) |
490 | // is transformed to |
491 | // addr = GEP(base, 0, gep_index) |
492 | replaceWithGEP(PreserveArrayIndexCalls, 1, 2); |
493 | replaceWithGEP(PreserveStructIndexCalls, 0, 1); |
494 | for (auto *Call : PreserveUnionIndexCalls) { |
495 | Call->replaceAllUsesWith(Call->getArgOperand(0)); |
496 | Call->eraseFromParent(); |
497 | } |
498 | |
499 | return Found; |
500 | } |
501 | |
502 | /// Check whether the access index chain is valid. We check |
503 | /// here because there may be type casts between two |
504 | /// access indexes. We want to ensure memory access still valid. |
505 | bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType, |
506 | uint32_t ParentAI, |
507 | const MDNode *ChildType) { |
508 | if (!ChildType) |
509 | return true; // preserve_field_info, no type comparison needed. |
510 | |
511 | const DIType *PType = stripQualifiers(cast<DIType>(ParentType)); |
512 | const DIType *CType = stripQualifiers(cast<DIType>(ChildType)); |
513 | |
514 | // Child is a derived/pointer type, which is due to type casting. |
515 | // Pointer type cannot be in the middle of chain. |
516 | if (isa<DIDerivedType>(CType)) |
517 | return false; |
518 | |
519 | // Parent is a pointer type. |
520 | if (const auto *PtrTy = dyn_cast<DIDerivedType>(PType)) { |
521 | if (PtrTy->getTag() != dwarf::DW_TAG_pointer_type) |
522 | return false; |
523 | return stripQualifiers(PtrTy->getBaseType()) == CType; |
524 | } |
525 | |
526 | // Otherwise, struct/union/array types |
527 | const auto *PTy = dyn_cast<DICompositeType>(PType); |
528 | const auto *CTy = dyn_cast<DICompositeType>(CType); |
529 | assert(PTy && CTy && "ParentType or ChildType is null or not composite")(static_cast <bool> (PTy && CTy && "ParentType or ChildType is null or not composite" ) ? void (0) : __assert_fail ("PTy && CTy && \"ParentType or ChildType is null or not composite\"" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 529, __extension__ __PRETTY_FUNCTION__)); |
530 | |
531 | uint32_t PTyTag = PTy->getTag(); |
532 | assert(PTyTag == dwarf::DW_TAG_array_type ||(static_cast <bool> (PTyTag == dwarf::DW_TAG_array_type || PTyTag == dwarf::DW_TAG_structure_type || PTyTag == dwarf ::DW_TAG_union_type) ? void (0) : __assert_fail ("PTyTag == dwarf::DW_TAG_array_type || PTyTag == dwarf::DW_TAG_structure_type || PTyTag == dwarf::DW_TAG_union_type" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 534, __extension__ __PRETTY_FUNCTION__)) |
533 | PTyTag == dwarf::DW_TAG_structure_type ||(static_cast <bool> (PTyTag == dwarf::DW_TAG_array_type || PTyTag == dwarf::DW_TAG_structure_type || PTyTag == dwarf ::DW_TAG_union_type) ? void (0) : __assert_fail ("PTyTag == dwarf::DW_TAG_array_type || PTyTag == dwarf::DW_TAG_structure_type || PTyTag == dwarf::DW_TAG_union_type" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 534, __extension__ __PRETTY_FUNCTION__)) |
534 | PTyTag == dwarf::DW_TAG_union_type)(static_cast <bool> (PTyTag == dwarf::DW_TAG_array_type || PTyTag == dwarf::DW_TAG_structure_type || PTyTag == dwarf ::DW_TAG_union_type) ? void (0) : __assert_fail ("PTyTag == dwarf::DW_TAG_array_type || PTyTag == dwarf::DW_TAG_structure_type || PTyTag == dwarf::DW_TAG_union_type" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 534, __extension__ __PRETTY_FUNCTION__)); |
535 | |
536 | uint32_t CTyTag = CTy->getTag(); |
537 | assert(CTyTag == dwarf::DW_TAG_array_type ||(static_cast <bool> (CTyTag == dwarf::DW_TAG_array_type || CTyTag == dwarf::DW_TAG_structure_type || CTyTag == dwarf ::DW_TAG_union_type) ? void (0) : __assert_fail ("CTyTag == dwarf::DW_TAG_array_type || CTyTag == dwarf::DW_TAG_structure_type || CTyTag == dwarf::DW_TAG_union_type" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 539, __extension__ __PRETTY_FUNCTION__)) |
538 | CTyTag == dwarf::DW_TAG_structure_type ||(static_cast <bool> (CTyTag == dwarf::DW_TAG_array_type || CTyTag == dwarf::DW_TAG_structure_type || CTyTag == dwarf ::DW_TAG_union_type) ? void (0) : __assert_fail ("CTyTag == dwarf::DW_TAG_array_type || CTyTag == dwarf::DW_TAG_structure_type || CTyTag == dwarf::DW_TAG_union_type" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 539, __extension__ __PRETTY_FUNCTION__)) |
539 | CTyTag == dwarf::DW_TAG_union_type)(static_cast <bool> (CTyTag == dwarf::DW_TAG_array_type || CTyTag == dwarf::DW_TAG_structure_type || CTyTag == dwarf ::DW_TAG_union_type) ? void (0) : __assert_fail ("CTyTag == dwarf::DW_TAG_array_type || CTyTag == dwarf::DW_TAG_structure_type || CTyTag == dwarf::DW_TAG_union_type" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 539, __extension__ __PRETTY_FUNCTION__)); |
540 | |
541 | // Multi dimensional arrays, base element should be the same |
542 | if (PTyTag == dwarf::DW_TAG_array_type && PTyTag == CTyTag) |
543 | return PTy->getBaseType() == CTy->getBaseType(); |
544 | |
545 | DIType *Ty; |
546 | if (PTyTag == dwarf::DW_TAG_array_type) |
547 | Ty = PTy->getBaseType(); |
548 | else |
549 | Ty = dyn_cast<DIType>(PTy->getElements()[ParentAI]); |
550 | |
551 | return dyn_cast<DICompositeType>(stripQualifiers(Ty)) == CTy; |
552 | } |
553 | |
554 | void BPFAbstractMemberAccess::traceAICall(CallInst *Call, |
555 | CallInfo &ParentInfo) { |
556 | for (User *U : Call->users()) { |
557 | Instruction *Inst = dyn_cast<Instruction>(U); |
558 | if (!Inst) |
559 | continue; |
560 | |
561 | if (auto *BI = dyn_cast<BitCastInst>(Inst)) { |
562 | traceBitCast(BI, Call, ParentInfo); |
563 | } else if (auto *CI = dyn_cast<CallInst>(Inst)) { |
564 | CallInfo ChildInfo; |
565 | |
566 | if (IsPreserveDIAccessIndexCall(CI, ChildInfo) && |
567 | IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex, |
568 | ChildInfo.Metadata)) { |
569 | AIChain[CI] = std::make_pair(Call, ParentInfo); |
570 | traceAICall(CI, ChildInfo); |
571 | } else { |
572 | BaseAICalls[Call] = ParentInfo; |
573 | } |
574 | } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { |
575 | if (GI->hasAllZeroIndices()) |
576 | traceGEP(GI, Call, ParentInfo); |
577 | else |
578 | BaseAICalls[Call] = ParentInfo; |
579 | } else { |
580 | BaseAICalls[Call] = ParentInfo; |
581 | } |
582 | } |
583 | } |
584 | |
585 | void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast, |
586 | CallInst *Parent, |
587 | CallInfo &ParentInfo) { |
588 | for (User *U : BitCast->users()) { |
589 | Instruction *Inst = dyn_cast<Instruction>(U); |
590 | if (!Inst) |
591 | continue; |
592 | |
593 | if (auto *BI = dyn_cast<BitCastInst>(Inst)) { |
594 | traceBitCast(BI, Parent, ParentInfo); |
595 | } else if (auto *CI = dyn_cast<CallInst>(Inst)) { |
596 | CallInfo ChildInfo; |
597 | if (IsPreserveDIAccessIndexCall(CI, ChildInfo) && |
598 | IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex, |
599 | ChildInfo.Metadata)) { |
600 | AIChain[CI] = std::make_pair(Parent, ParentInfo); |
601 | traceAICall(CI, ChildInfo); |
602 | } else { |
603 | BaseAICalls[Parent] = ParentInfo; |
604 | } |
605 | } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { |
606 | if (GI->hasAllZeroIndices()) |
607 | traceGEP(GI, Parent, ParentInfo); |
608 | else |
609 | BaseAICalls[Parent] = ParentInfo; |
610 | } else { |
611 | BaseAICalls[Parent] = ParentInfo; |
612 | } |
613 | } |
614 | } |
615 | |
616 | void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent, |
617 | CallInfo &ParentInfo) { |
618 | for (User *U : GEP->users()) { |
619 | Instruction *Inst = dyn_cast<Instruction>(U); |
620 | if (!Inst) |
621 | continue; |
622 | |
623 | if (auto *BI = dyn_cast<BitCastInst>(Inst)) { |
624 | traceBitCast(BI, Parent, ParentInfo); |
625 | } else if (auto *CI = dyn_cast<CallInst>(Inst)) { |
626 | CallInfo ChildInfo; |
627 | if (IsPreserveDIAccessIndexCall(CI, ChildInfo) && |
628 | IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex, |
629 | ChildInfo.Metadata)) { |
630 | AIChain[CI] = std::make_pair(Parent, ParentInfo); |
631 | traceAICall(CI, ChildInfo); |
632 | } else { |
633 | BaseAICalls[Parent] = ParentInfo; |
634 | } |
635 | } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { |
636 | if (GI->hasAllZeroIndices()) |
637 | traceGEP(GI, Parent, ParentInfo); |
638 | else |
639 | BaseAICalls[Parent] = ParentInfo; |
640 | } else { |
641 | BaseAICalls[Parent] = ParentInfo; |
642 | } |
643 | } |
644 | } |
645 | |
646 | void BPFAbstractMemberAccess::collectAICallChains(Function &F) { |
647 | AIChain.clear(); |
648 | BaseAICalls.clear(); |
649 | |
650 | for (auto &BB : F) |
651 | for (auto &I : BB) { |
652 | CallInfo CInfo; |
653 | auto *Call = dyn_cast<CallInst>(&I); |
654 | if (!IsPreserveDIAccessIndexCall(Call, CInfo) || |
655 | AIChain.find(Call) != AIChain.end()) |
656 | continue; |
657 | |
658 | traceAICall(Call, CInfo); |
659 | } |
660 | } |
661 | |
662 | uint64_t BPFAbstractMemberAccess::getConstant(const Value *IndexValue) { |
663 | const ConstantInt *CV = dyn_cast<ConstantInt>(IndexValue); |
664 | assert(CV)(static_cast <bool> (CV) ? void (0) : __assert_fail ("CV" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 664, __extension__ __PRETTY_FUNCTION__)); |
665 | return CV->getValue().getZExtValue(); |
666 | } |
667 | |
668 | /// Get the start and the end of storage offset for \p MemberTy. |
669 | void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy, |
670 | Align RecordAlignment, |
671 | uint32_t &StartBitOffset, |
672 | uint32_t &EndBitOffset) { |
673 | uint32_t MemberBitSize = MemberTy->getSizeInBits(); |
674 | uint32_t MemberBitOffset = MemberTy->getOffsetInBits(); |
675 | |
676 | if (RecordAlignment > 8) { |
677 | // If the Bits are within an aligned 8-byte, set the RecordAlignment |
678 | // to 8, other report the fatal error. |
679 | if (MemberBitOffset / 64 != (MemberBitOffset + MemberBitSize) / 64) |
680 | report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, " |
681 | "requiring too big alignment"); |
682 | RecordAlignment = Align(8); |
683 | } |
684 | |
685 | uint32_t AlignBits = RecordAlignment.value() * 8; |
686 | if (MemberBitSize > AlignBits) |
687 | report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, " |
688 | "bitfield size greater than record alignment"); |
689 | |
690 | StartBitOffset = MemberBitOffset & ~(AlignBits - 1); |
691 | if ((StartBitOffset + AlignBits) < (MemberBitOffset + MemberBitSize)) |
692 | report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, " |
693 | "cross alignment boundary"); |
694 | EndBitOffset = StartBitOffset + AlignBits; |
695 | } |
696 | |
697 | uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind, |
698 | DICompositeType *CTy, |
699 | uint32_t AccessIndex, |
700 | uint32_t PatchImm, |
701 | MaybeAlign RecordAlignment) { |
702 | if (InfoKind == BPFCoreSharedInfo::FIELD_EXISTENCE) |
703 | return 1; |
704 | |
705 | uint32_t Tag = CTy->getTag(); |
706 | if (InfoKind == BPFCoreSharedInfo::FIELD_BYTE_OFFSET) { |
707 | if (Tag == dwarf::DW_TAG_array_type) { |
708 | auto *EltTy = stripQualifiers(CTy->getBaseType()); |
709 | PatchImm += AccessIndex * calcArraySize(CTy, 1) * |
710 | (EltTy->getSizeInBits() >> 3); |
711 | } else if (Tag == dwarf::DW_TAG_structure_type) { |
712 | auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); |
713 | if (!MemberTy->isBitField()) { |
714 | PatchImm += MemberTy->getOffsetInBits() >> 3; |
715 | } else { |
716 | unsigned SBitOffset, NextSBitOffset; |
717 | GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, |
718 | NextSBitOffset); |
719 | PatchImm += SBitOffset >> 3; |
720 | } |
721 | } |
722 | return PatchImm; |
723 | } |
724 | |
725 | if (InfoKind == BPFCoreSharedInfo::FIELD_BYTE_SIZE) { |
726 | if (Tag == dwarf::DW_TAG_array_type) { |
727 | auto *EltTy = stripQualifiers(CTy->getBaseType()); |
728 | return calcArraySize(CTy, 1) * (EltTy->getSizeInBits() >> 3); |
729 | } else { |
730 | auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); |
731 | uint32_t SizeInBits = MemberTy->getSizeInBits(); |
732 | if (!MemberTy->isBitField()) |
733 | return SizeInBits >> 3; |
734 | |
735 | unsigned SBitOffset, NextSBitOffset; |
736 | GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, |
737 | NextSBitOffset); |
738 | SizeInBits = NextSBitOffset - SBitOffset; |
739 | if (SizeInBits & (SizeInBits - 1)) |
740 | report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info"); |
741 | return SizeInBits >> 3; |
742 | } |
743 | } |
744 | |
745 | if (InfoKind == BPFCoreSharedInfo::FIELD_SIGNEDNESS) { |
746 | const DIType *BaseTy; |
747 | if (Tag == dwarf::DW_TAG_array_type) { |
748 | // Signedness only checked when final array elements are accessed. |
749 | if (CTy->getElements().size() != 1) |
750 | report_fatal_error("Invalid array expression for llvm.bpf.preserve.field.info"); |
751 | BaseTy = stripQualifiers(CTy->getBaseType()); |
752 | } else { |
753 | auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); |
754 | BaseTy = stripQualifiers(MemberTy->getBaseType()); |
755 | } |
756 | |
757 | // Only basic types and enum types have signedness. |
758 | const auto *BTy = dyn_cast<DIBasicType>(BaseTy); |
759 | while (!BTy) { |
760 | const auto *CompTy = dyn_cast<DICompositeType>(BaseTy); |
761 | // Report an error if the field expression does not have signedness. |
762 | if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type) |
763 | report_fatal_error("Invalid field expression for llvm.bpf.preserve.field.info"); |
764 | BaseTy = stripQualifiers(CompTy->getBaseType()); |
765 | BTy = dyn_cast<DIBasicType>(BaseTy); |
766 | } |
767 | uint32_t Encoding = BTy->getEncoding(); |
768 | return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char); |
769 | } |
770 | |
771 | if (InfoKind == BPFCoreSharedInfo::FIELD_LSHIFT_U64) { |
772 | // The value is loaded into a value with FIELD_BYTE_SIZE size, |
773 | // and then zero or sign extended to U64. |
774 | // FIELD_LSHIFT_U64 and FIELD_RSHIFT_U64 are operations |
775 | // to extract the original value. |
776 | const Triple &Triple = TM->getTargetTriple(); |
777 | DIDerivedType *MemberTy = nullptr; |
778 | bool IsBitField = false; |
779 | uint32_t SizeInBits; |
780 | |
781 | if (Tag == dwarf::DW_TAG_array_type) { |
782 | auto *EltTy = stripQualifiers(CTy->getBaseType()); |
783 | SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits(); |
784 | } else { |
785 | MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); |
786 | SizeInBits = MemberTy->getSizeInBits(); |
787 | IsBitField = MemberTy->isBitField(); |
788 | } |
789 | |
790 | if (!IsBitField) { |
791 | if (SizeInBits > 64) |
792 | report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); |
793 | return 64 - SizeInBits; |
794 | } |
795 | |
796 | unsigned SBitOffset, NextSBitOffset; |
797 | GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset); |
798 | if (NextSBitOffset - SBitOffset > 64) |
799 | report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); |
800 | |
801 | unsigned OffsetInBits = MemberTy->getOffsetInBits(); |
802 | if (Triple.getArch() == Triple::bpfel) |
803 | return SBitOffset + 64 - OffsetInBits - SizeInBits; |
804 | else |
805 | return OffsetInBits + 64 - NextSBitOffset; |
806 | } |
807 | |
808 | if (InfoKind == BPFCoreSharedInfo::FIELD_RSHIFT_U64) { |
809 | DIDerivedType *MemberTy = nullptr; |
810 | bool IsBitField = false; |
811 | uint32_t SizeInBits; |
812 | if (Tag == dwarf::DW_TAG_array_type) { |
813 | auto *EltTy = stripQualifiers(CTy->getBaseType()); |
814 | SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits(); |
815 | } else { |
816 | MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); |
817 | SizeInBits = MemberTy->getSizeInBits(); |
818 | IsBitField = MemberTy->isBitField(); |
819 | } |
820 | |
821 | if (!IsBitField) { |
822 | if (SizeInBits > 64) |
823 | report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); |
824 | return 64 - SizeInBits; |
825 | } |
826 | |
827 | unsigned SBitOffset, NextSBitOffset; |
828 | GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset); |
829 | if (NextSBitOffset - SBitOffset > 64) |
830 | report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); |
831 | |
832 | return 64 - SizeInBits; |
833 | } |
834 | |
835 | llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind")::llvm::llvm_unreachable_internal("Unknown llvm.bpf.preserve.field.info info kind" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 835); |
836 | } |
837 | |
838 | bool BPFAbstractMemberAccess::HasPreserveFieldInfoCall(CallInfoStack &CallStack) { |
839 | // This is called in error return path, no need to maintain CallStack. |
840 | while (CallStack.size()) { |
841 | auto StackElem = CallStack.top(); |
842 | if (StackElem.second.Kind == BPFPreserveFieldInfoAI) |
843 | return true; |
844 | CallStack.pop(); |
845 | } |
846 | return false; |
847 | } |
848 | |
849 | /// Compute the base of the whole preserve_* intrinsics chains, i.e., the base |
850 | /// pointer of the first preserve_*_access_index call, and construct the access |
851 | /// string, which will be the name of a global variable. |
852 | Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call, |
853 | CallInfo &CInfo, |
854 | std::string &AccessKey, |
855 | MDNode *&TypeMeta) { |
856 | Value *Base = nullptr; |
857 | std::string TypeName; |
858 | CallInfoStack CallStack; |
859 | |
860 | // Put the access chain into a stack with the top as the head of the chain. |
861 | while (Call) { |
862 | CallStack.push(std::make_pair(Call, CInfo)); |
863 | CInfo = AIChain[Call].second; |
864 | Call = AIChain[Call].first; |
865 | } |
866 | |
867 | // The access offset from the base of the head of chain is also |
868 | // calculated here as all debuginfo types are available. |
869 | |
870 | // Get type name and calculate the first index. |
871 | // We only want to get type name from typedef, structure or union. |
872 | // If user wants a relocation like |
873 | // int *p; ... __builtin_preserve_access_index(&p[4]) ... |
874 | // or |
875 | // int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ... |
876 | // we will skip them. |
877 | uint32_t FirstIndex = 0; |
878 | uint32_t PatchImm = 0; // AccessOffset or the requested field info |
879 | uint32_t InfoKind = BPFCoreSharedInfo::FIELD_BYTE_OFFSET; |
880 | while (CallStack.size()) { |
881 | auto StackElem = CallStack.top(); |
882 | Call = StackElem.first; |
Value stored to 'Call' is never read | |
883 | CInfo = StackElem.second; |
884 | |
885 | if (!Base) |
886 | Base = CInfo.Base; |
887 | |
888 | DIType *PossibleTypeDef = stripQualifiers(cast<DIType>(CInfo.Metadata), |
889 | false); |
890 | DIType *Ty = stripQualifiers(PossibleTypeDef); |
891 | if (CInfo.Kind == BPFPreserveUnionAI || |
892 | CInfo.Kind == BPFPreserveStructAI) { |
893 | // struct or union type. If the typedef is in the metadata, always |
894 | // use the typedef. |
895 | TypeName = std::string(PossibleTypeDef->getName()); |
896 | TypeMeta = PossibleTypeDef; |
897 | PatchImm += FirstIndex * (Ty->getSizeInBits() >> 3); |
898 | break; |
899 | } |
900 | |
901 | assert(CInfo.Kind == BPFPreserveArrayAI)(static_cast <bool> (CInfo.Kind == BPFPreserveArrayAI) ? void (0) : __assert_fail ("CInfo.Kind == BPFPreserveArrayAI" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 901, __extension__ __PRETTY_FUNCTION__)); |
902 | |
903 | // Array entries will always be consumed for accumulative initial index. |
904 | CallStack.pop(); |
905 | |
906 | // BPFPreserveArrayAI |
907 | uint64_t AccessIndex = CInfo.AccessIndex; |
908 | |
909 | DIType *BaseTy = nullptr; |
910 | bool CheckElemType = false; |
911 | if (const auto *CTy = dyn_cast<DICompositeType>(Ty)) { |
912 | // array type |
913 | assert(CTy->getTag() == dwarf::DW_TAG_array_type)(static_cast <bool> (CTy->getTag() == dwarf::DW_TAG_array_type ) ? void (0) : __assert_fail ("CTy->getTag() == dwarf::DW_TAG_array_type" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 913, __extension__ __PRETTY_FUNCTION__)); |
914 | |
915 | |
916 | FirstIndex += AccessIndex * calcArraySize(CTy, 1); |
917 | BaseTy = stripQualifiers(CTy->getBaseType()); |
918 | CheckElemType = CTy->getElements().size() == 1; |
919 | } else { |
920 | // pointer type |
921 | auto *DTy = cast<DIDerivedType>(Ty); |
922 | assert(DTy->getTag() == dwarf::DW_TAG_pointer_type)(static_cast <bool> (DTy->getTag() == dwarf::DW_TAG_pointer_type ) ? void (0) : __assert_fail ("DTy->getTag() == dwarf::DW_TAG_pointer_type" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 922, __extension__ __PRETTY_FUNCTION__)); |
923 | |
924 | BaseTy = stripQualifiers(DTy->getBaseType()); |
925 | CTy = dyn_cast<DICompositeType>(BaseTy); |
926 | if (!CTy) { |
927 | CheckElemType = true; |
928 | } else if (CTy->getTag() != dwarf::DW_TAG_array_type) { |
929 | FirstIndex += AccessIndex; |
930 | CheckElemType = true; |
931 | } else { |
932 | FirstIndex += AccessIndex * calcArraySize(CTy, 0); |
933 | } |
934 | } |
935 | |
936 | if (CheckElemType) { |
937 | auto *CTy = dyn_cast<DICompositeType>(BaseTy); |
938 | if (!CTy) { |
939 | if (HasPreserveFieldInfoCall(CallStack)) |
940 | report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic"); |
941 | return nullptr; |
942 | } |
943 | |
944 | unsigned CTag = CTy->getTag(); |
945 | if (CTag == dwarf::DW_TAG_structure_type || CTag == dwarf::DW_TAG_union_type) { |
946 | TypeName = std::string(CTy->getName()); |
947 | } else { |
948 | if (HasPreserveFieldInfoCall(CallStack)) |
949 | report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic"); |
950 | return nullptr; |
951 | } |
952 | TypeMeta = CTy; |
953 | PatchImm += FirstIndex * (CTy->getSizeInBits() >> 3); |
954 | break; |
955 | } |
956 | } |
957 | assert(TypeName.size())(static_cast <bool> (TypeName.size()) ? void (0) : __assert_fail ("TypeName.size()", "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp" , 957, __extension__ __PRETTY_FUNCTION__)); |
958 | AccessKey += std::to_string(FirstIndex); |
959 | |
960 | // Traverse the rest of access chain to complete offset calculation |
961 | // and access key construction. |
962 | while (CallStack.size()) { |
963 | auto StackElem = CallStack.top(); |
964 | CInfo = StackElem.second; |
965 | CallStack.pop(); |
966 | |
967 | if (CInfo.Kind == BPFPreserveFieldInfoAI) { |
968 | InfoKind = CInfo.AccessIndex; |
969 | if (InfoKind == BPFCoreSharedInfo::FIELD_EXISTENCE) |
970 | PatchImm = 1; |
971 | break; |
972 | } |
973 | |
974 | // If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI, |
975 | // the action will be extracting field info. |
976 | if (CallStack.size()) { |
977 | auto StackElem2 = CallStack.top(); |
978 | CallInfo CInfo2 = StackElem2.second; |
979 | if (CInfo2.Kind == BPFPreserveFieldInfoAI) { |
980 | InfoKind = CInfo2.AccessIndex; |
981 | assert(CallStack.size() == 1)(static_cast <bool> (CallStack.size() == 1) ? void (0) : __assert_fail ("CallStack.size() == 1", "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp" , 981, __extension__ __PRETTY_FUNCTION__)); |
982 | } |
983 | } |
984 | |
985 | // Access Index |
986 | uint64_t AccessIndex = CInfo.AccessIndex; |
987 | AccessKey += ":" + std::to_string(AccessIndex); |
988 | |
989 | MDNode *MDN = CInfo.Metadata; |
990 | // At this stage, it cannot be pointer type. |
991 | auto *CTy = cast<DICompositeType>(stripQualifiers(cast<DIType>(MDN))); |
992 | PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex, PatchImm, |
993 | CInfo.RecordAlignment); |
994 | } |
995 | |
996 | // Access key is the |
997 | // "llvm." + type name + ":" + reloc type + ":" + patched imm + "$" + |
998 | // access string, |
999 | // uniquely identifying one relocation. |
1000 | // The prefix "llvm." indicates this is a temporary global, which should |
1001 | // not be emitted to ELF file. |
1002 | AccessKey = "llvm." + TypeName + ":" + std::to_string(InfoKind) + ":" + |
1003 | std::to_string(PatchImm) + "$" + AccessKey; |
1004 | |
1005 | return Base; |
1006 | } |
1007 | |
1008 | MDNode *BPFAbstractMemberAccess::computeAccessKey(CallInst *Call, |
1009 | CallInfo &CInfo, |
1010 | std::string &AccessKey, |
1011 | bool &IsInt32Ret) { |
1012 | DIType *Ty = stripQualifiers(cast<DIType>(CInfo.Metadata), false); |
1013 | assert(!Ty->getName().empty())(static_cast <bool> (!Ty->getName().empty()) ? void ( 0) : __assert_fail ("!Ty->getName().empty()", "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp" , 1013, __extension__ __PRETTY_FUNCTION__)); |
1014 | |
1015 | int64_t PatchImm; |
1016 | std::string AccessStr("0"); |
1017 | if (CInfo.AccessIndex == BPFCoreSharedInfo::TYPE_EXISTENCE || |
1018 | CInfo.AccessIndex == BPFCoreSharedInfo::TYPE_MATCH) { |
1019 | PatchImm = 1; |
1020 | } else if (CInfo.AccessIndex == BPFCoreSharedInfo::TYPE_SIZE) { |
1021 | // typedef debuginfo type has size 0, get the eventual base type. |
1022 | DIType *BaseTy = stripQualifiers(Ty, true); |
1023 | PatchImm = BaseTy->getSizeInBits() / 8; |
1024 | } else { |
1025 | // ENUM_VALUE_EXISTENCE and ENUM_VALUE |
1026 | IsInt32Ret = false; |
1027 | |
1028 | // The argument could be a global variable or a getelementptr with base to |
1029 | // a global variable depending on whether the clang option `opaque-options` |
1030 | // is set or not. |
1031 | const GlobalVariable *GV = |
1032 | cast<GlobalVariable>(Call->getArgOperand(1)->stripPointerCasts()); |
1033 | assert(GV->hasInitializer())(static_cast <bool> (GV->hasInitializer()) ? void (0 ) : __assert_fail ("GV->hasInitializer()", "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp" , 1033, __extension__ __PRETTY_FUNCTION__)); |
1034 | const ConstantDataArray *DA = cast<ConstantDataArray>(GV->getInitializer()); |
1035 | assert(DA->isString())(static_cast <bool> (DA->isString()) ? void (0) : __assert_fail ("DA->isString()", "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp" , 1035, __extension__ __PRETTY_FUNCTION__)); |
1036 | StringRef ValueStr = DA->getAsString(); |
1037 | |
1038 | // ValueStr format: <EnumeratorStr>:<Value> |
1039 | size_t Separator = ValueStr.find_first_of(':'); |
1040 | StringRef EnumeratorStr = ValueStr.substr(0, Separator); |
1041 | |
1042 | // Find enumerator index in the debuginfo |
1043 | DIType *BaseTy = stripQualifiers(Ty, true); |
1044 | const auto *CTy = cast<DICompositeType>(BaseTy); |
1045 | assert(CTy->getTag() == dwarf::DW_TAG_enumeration_type)(static_cast <bool> (CTy->getTag() == dwarf::DW_TAG_enumeration_type ) ? void (0) : __assert_fail ("CTy->getTag() == dwarf::DW_TAG_enumeration_type" , "llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp", 1045, __extension__ __PRETTY_FUNCTION__)); |
1046 | int EnumIndex = 0; |
1047 | for (const auto Element : CTy->getElements()) { |
1048 | const auto *Enum = cast<DIEnumerator>(Element); |
1049 | if (Enum->getName() == EnumeratorStr) { |
1050 | AccessStr = std::to_string(EnumIndex); |
1051 | break; |
1052 | } |
1053 | EnumIndex++; |
1054 | } |
1055 | |
1056 | if (CInfo.AccessIndex == BPFCoreSharedInfo::ENUM_VALUE) { |
1057 | StringRef EValueStr = ValueStr.substr(Separator + 1); |
1058 | PatchImm = std::stoll(std::string(EValueStr)); |
1059 | } else { |
1060 | PatchImm = 1; |
1061 | } |
1062 | } |
1063 | |
1064 | AccessKey = "llvm." + Ty->getName().str() + ":" + |
1065 | std::to_string(CInfo.AccessIndex) + std::string(":") + |
1066 | std::to_string(PatchImm) + std::string("$") + AccessStr; |
1067 | |
1068 | return Ty; |
1069 | } |
1070 | |
1071 | /// Call/Kind is the base preserve_*_access_index() call. Attempts to do |
1072 | /// transformation to a chain of relocable GEPs. |
1073 | bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call, |
1074 | CallInfo &CInfo) { |
1075 | std::string AccessKey; |
1076 | MDNode *TypeMeta; |
1077 | Value *Base = nullptr; |
1078 | bool IsInt32Ret; |
1079 | |
1080 | IsInt32Ret = CInfo.Kind == BPFPreserveFieldInfoAI; |
1081 | if (CInfo.Kind == BPFPreserveFieldInfoAI && CInfo.Metadata) { |
1082 | TypeMeta = computeAccessKey(Call, CInfo, AccessKey, IsInt32Ret); |
1083 | } else { |
1084 | Base = computeBaseAndAccessKey(Call, CInfo, AccessKey, TypeMeta); |
1085 | if (!Base) |
1086 | return false; |
1087 | } |
1088 | |
1089 | BasicBlock *BB = Call->getParent(); |
1090 | GlobalVariable *GV; |
1091 | |
1092 | if (GEPGlobals.find(AccessKey) == GEPGlobals.end()) { |
1093 | IntegerType *VarType; |
1094 | if (IsInt32Ret) |
1095 | VarType = Type::getInt32Ty(BB->getContext()); // 32bit return value |
1096 | else |
1097 | VarType = Type::getInt64Ty(BB->getContext()); // 64bit ptr or enum value |
1098 | |
1099 | GV = new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage, |
1100 | nullptr, AccessKey); |
1101 | GV->addAttribute(BPFCoreSharedInfo::AmaAttr); |
1102 | GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta); |
1103 | GEPGlobals[AccessKey] = GV; |
1104 | } else { |
1105 | GV = GEPGlobals[AccessKey]; |
1106 | } |
1107 | |
1108 | if (CInfo.Kind == BPFPreserveFieldInfoAI) { |
1109 | // Load the global variable which represents the returned field info. |
1110 | LoadInst *LDInst; |
1111 | if (IsInt32Ret) |
1112 | LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV, "", Call); |
1113 | else |
1114 | LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", Call); |
1115 | |
1116 | Instruction *PassThroughInst = |
1117 | BPFCoreSharedInfo::insertPassThrough(M, BB, LDInst, Call); |
1118 | Call->replaceAllUsesWith(PassThroughInst); |
1119 | Call->eraseFromParent(); |
1120 | return true; |
1121 | } |
1122 | |
1123 | // For any original GEP Call and Base %2 like |
1124 | // %4 = bitcast %struct.net_device** %dev1 to i64* |
1125 | // it is transformed to: |
1126 | // %6 = load llvm.sk_buff:0:50$0:0:0:2:0 |
1127 | // %7 = bitcast %struct.sk_buff* %2 to i8* |
1128 | // %8 = getelementptr i8, i8* %7, %6 |
1129 | // %9 = bitcast i8* %8 to i64* |
1130 | // using %9 instead of %4 |
1131 | // The original Call inst is removed. |
1132 | |
1133 | // Load the global variable. |
1134 | auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", Call); |
1135 | |
1136 | // Generate a BitCast |
1137 | auto *BCInst = new BitCastInst(Base, Type::getInt8PtrTy(BB->getContext())); |
1138 | BCInst->insertBefore(Call); |
1139 | |
1140 | // Generate a GetElementPtr |
1141 | auto *GEP = GetElementPtrInst::Create(Type::getInt8Ty(BB->getContext()), |
1142 | BCInst, LDInst); |
1143 | GEP->insertBefore(Call); |
1144 | |
1145 | // Generate a BitCast |
1146 | auto *BCInst2 = new BitCastInst(GEP, Call->getType()); |
1147 | BCInst2->insertBefore(Call); |
1148 | |
1149 | // For the following code, |
1150 | // Block0: |
1151 | // ... |
1152 | // if (...) goto Block1 else ... |
1153 | // Block1: |
1154 | // %6 = load llvm.sk_buff:0:50$0:0:0:2:0 |
1155 | // %7 = bitcast %struct.sk_buff* %2 to i8* |
1156 | // %8 = getelementptr i8, i8* %7, %6 |
1157 | // ... |
1158 | // goto CommonExit |
1159 | // Block2: |
1160 | // ... |
1161 | // if (...) goto Block3 else ... |
1162 | // Block3: |
1163 | // %6 = load llvm.bpf_map:0:40$0:0:0:2:0 |
1164 | // %7 = bitcast %struct.sk_buff* %2 to i8* |
1165 | // %8 = getelementptr i8, i8* %7, %6 |
1166 | // ... |
1167 | // goto CommonExit |
1168 | // CommonExit |
1169 | // SimplifyCFG may generate: |
1170 | // Block0: |
1171 | // ... |
1172 | // if (...) goto Block_Common else ... |
1173 | // Block2: |
1174 | // ... |
1175 | // if (...) goto Block_Common else ... |
1176 | // Block_Common: |
1177 | // PHI = [llvm.sk_buff:0:50$0:0:0:2:0, llvm.bpf_map:0:40$0:0:0:2:0] |
1178 | // %6 = load PHI |
1179 | // %7 = bitcast %struct.sk_buff* %2 to i8* |
1180 | // %8 = getelementptr i8, i8* %7, %6 |
1181 | // ... |
1182 | // goto CommonExit |
1183 | // For the above code, we cannot perform proper relocation since |
1184 | // "load PHI" has two possible relocations. |
1185 | // |
1186 | // To prevent above tail merging, we use __builtin_bpf_passthrough() |
1187 | // where one of its parameters is a seq_num. Since two |
1188 | // __builtin_bpf_passthrough() funcs will always have different seq_num, |
1189 | // tail merging cannot happen. The __builtin_bpf_passthrough() will be |
1190 | // removed in the beginning of Target IR passes. |
1191 | // |
1192 | // This approach is also used in other places when global var |
1193 | // representing a relocation is used. |
1194 | Instruction *PassThroughInst = |
1195 | BPFCoreSharedInfo::insertPassThrough(M, BB, BCInst2, Call); |
1196 | Call->replaceAllUsesWith(PassThroughInst); |
1197 | Call->eraseFromParent(); |
1198 | |
1199 | return true; |
1200 | } |
1201 | |
1202 | bool BPFAbstractMemberAccess::doTransformation(Function &F) { |
1203 | bool Transformed = false; |
1204 | |
1205 | // Collect PreserveDIAccessIndex Intrinsic call chains. |
1206 | // The call chains will be used to generate the access |
1207 | // patterns similar to GEP. |
1208 | collectAICallChains(F); |
1209 | |
1210 | for (auto &C : BaseAICalls) |
1211 | Transformed = transformGEPChain(C.first, C.second) || Transformed; |
1212 | |
1213 | return removePreserveAccessIndexIntrinsic(F) || Transformed; |
1214 | } |
1215 | |
1216 | PreservedAnalyses |
1217 | BPFAbstractMemberAccessPass::run(Function &F, FunctionAnalysisManager &AM) { |
1218 | return BPFAbstractMemberAccess(TM).run(F) ? PreservedAnalyses::none() |
1219 | : PreservedAnalyses::all(); |
1220 | } |