LLVM  9.0.0svn
HWAddressSanitizer.cpp
Go to the documentation of this file.
1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file is a part of HWAddressSanitizer, an address sanity checker
11 /// based on tagged addressing.
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/ADT/SmallVector.h"
15 #include "llvm/ADT/StringExtras.h"
16 #include "llvm/ADT/StringRef.h"
17 #include "llvm/ADT/Triple.h"
18 #include "llvm/IR/Attributes.h"
19 #include "llvm/IR/BasicBlock.h"
20 #include "llvm/IR/Constant.h"
21 #include "llvm/IR/Constants.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/IR/Function.h"
25 #include "llvm/IR/IRBuilder.h"
26 #include "llvm/IR/InlineAsm.h"
27 #include "llvm/IR/InstVisitor.h"
28 #include "llvm/IR/Instruction.h"
29 #include "llvm/IR/Instructions.h"
30 #include "llvm/IR/IntrinsicInst.h"
31 #include "llvm/IR/Intrinsics.h"
32 #include "llvm/IR/LLVMContext.h"
33 #include "llvm/IR/MDBuilder.h"
34 #include "llvm/IR/Module.h"
35 #include "llvm/IR/Type.h"
36 #include "llvm/IR/Value.h"
37 #include "llvm/Pass.h"
38 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/Debug.h"
46 #include <sstream>
47 
48 using namespace llvm;
49 
50 #define DEBUG_TYPE "hwasan"
51 
52 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
53 static const char *const kHwasanInitName = "__hwasan_init";
54 
55 static const char *const kHwasanShadowMemoryDynamicAddress =
56  "__hwasan_shadow_memory_dynamic_address";
57 
58 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
59 static const size_t kNumberOfAccessSizes = 5;
60 
61 static const size_t kDefaultShadowScale = 4;
62 static const uint64_t kDynamicShadowSentinel =
64 static const unsigned kPointerTagShift = 56;
65 
66 static const unsigned kShadowBaseAlignment = 32;
67 
69  "hwasan-memory-access-callback-prefix",
70  cl::desc("Prefix for memory access callbacks"), cl::Hidden,
71  cl::init("__hwasan_"));
72 
73 static cl::opt<bool>
74  ClInstrumentWithCalls("hwasan-instrument-with-calls",
75  cl::desc("instrument reads and writes with callbacks"),
76  cl::Hidden, cl::init(false));
77 
78 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
79  cl::desc("instrument read instructions"),
80  cl::Hidden, cl::init(true));
81 
83  "hwasan-instrument-writes", cl::desc("instrument write instructions"),
84  cl::Hidden, cl::init(true));
85 
87  "hwasan-instrument-atomics",
88  cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
89  cl::init(true));
90 
92  "hwasan-recover",
93  cl::desc("Enable recovery mode (continue-after-error)."),
94  cl::Hidden, cl::init(false));
95 
96 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
97  cl::desc("instrument stack (allocas)"),
98  cl::Hidden, cl::init(true));
99 
101  "hwasan-uar-retag-to-zero",
102  cl::desc("Clear alloca tags before returning from the function to allow "
103  "non-instrumented and instrumented function calls mix. When set "
104  "to false, allocas are retagged before returning from the "
105  "function to detect use after return."),
106  cl::Hidden, cl::init(true));
107 
109  "hwasan-generate-tags-with-calls",
110  cl::desc("generate new tags with runtime library calls"), cl::Hidden,
111  cl::init(false));
112 
114  "hwasan-match-all-tag",
115  cl::desc("don't report bad accesses via pointers with this tag"),
116  cl::Hidden, cl::init(-1));
117 
119  "hwasan-kernel",
120  cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
121  cl::Hidden, cl::init(false));
122 
123 // These flags allow to change the shadow mapping and control how shadow memory
124 // is accessed. The shadow mapping looks like:
125 // Shadow = (Mem >> scale) + offset
126 
128  "hwasan-mapping-offset",
129  cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"), cl::Hidden,
130  cl::init(0));
131 
132 static cl::opt<bool>
133  ClWithIfunc("hwasan-with-ifunc",
134  cl::desc("Access dynamic shadow through an ifunc global on "
135  "platforms that support this"),
136  cl::Hidden, cl::init(false));
137 
138 static cl::opt<bool> ClWithTls(
139  "hwasan-with-tls",
140  cl::desc("Access dynamic shadow through an thread-local pointer on "
141  "platforms that support this"),
142  cl::Hidden, cl::init(true));
143 
144 static cl::opt<bool>
145  ClRecordStackHistory("hwasan-record-stack-history",
146  cl::desc("Record stack frames with tagged allocations "
147  "in a thread-local ring buffer"),
148  cl::Hidden, cl::init(true));
149 static cl::opt<bool>
150  ClCreateFrameDescriptions("hwasan-create-frame-descriptions",
151  cl::desc("create static frame descriptions"),
152  cl::Hidden, cl::init(true));
153 
154 static cl::opt<bool>
155  ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
156  cl::desc("instrument memory intrinsics"),
157  cl::Hidden, cl::init(true));
158 
159 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
160  cl::desc("inline all checks"),
161  cl::Hidden, cl::init(false));
162 
163 static cl::opt<bool> ClAllowIfunc("hwasan-allow-ifunc",
164  cl::desc("allow the use of ifunc"),
165  cl::Hidden, cl::init(false));
166 
167 namespace {
168 
169 /// An instrumentation pass implementing detection of addressability bugs
170 /// using tagged pointers.
171 class HWAddressSanitizer : public FunctionPass {
172 public:
173  // Pass identification, replacement for typeid.
174  static char ID;
175 
176  explicit HWAddressSanitizer(bool CompileKernel = false, bool Recover = false)
177  : FunctionPass(ID) {
178  this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
179  this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
180  ClEnableKhwasan : CompileKernel;
181  }
182 
183  StringRef getPassName() const override { return "HWAddressSanitizer"; }
184 
185  bool runOnFunction(Function &F) override;
186  bool doInitialization(Module &M) override;
187 
188  void initializeCallbacks(Module &M);
189 
190  Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
191  Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
192 
193  void untagPointerOperand(Instruction *I, Value *Addr);
194  Value *shadowBase();
195  Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
196  void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
197  unsigned AccessSizeIndex,
198  Instruction *InsertBefore);
199  void instrumentMemIntrinsic(MemIntrinsic *MI);
200  bool instrumentMemAccess(Instruction *I);
201  Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
202  uint64_t *TypeSize, unsigned *Alignment,
203  Value **MaybeMask);
204 
205  bool isInterestingAlloca(const AllocaInst &AI);
206  bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag);
207  Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
208  Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
209  bool instrumentStack(SmallVectorImpl<AllocaInst *> &Allocas,
210  SmallVectorImpl<Instruction *> &RetVec, Value *StackTag);
211  Value *getNextTagWithCall(IRBuilder<> &IRB);
212  Value *getStackBaseTag(IRBuilder<> &IRB);
213  Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
214  unsigned AllocaNo);
215  Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
216 
217  Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
218  Value *emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
219 
220 private:
221  LLVMContext *C;
222  std::string CurModuleUniqueId;
223  Triple TargetTriple;
224  FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
225 
226  // Frame description is a way to pass names/sizes of local variables
227  // to the run-time w/o adding extra executable code in every function.
228  // We do this by creating a separate section with {PC,Descr} pairs and passing
229  // the section beg/end to __hwasan_init_frames() at module init time.
230  std::string createFrameString(ArrayRef<AllocaInst*> Allocas);
231  void createFrameGlobal(Function &F, const std::string &FrameString);
232  // Get the section name for frame descriptions. Currently ELF-only.
233  const char *getFrameSection() { return "__hwasan_frames"; }
234  const char *getFrameSectionBeg() { return "__start___hwasan_frames"; }
235  const char *getFrameSectionEnd() { return "__stop___hwasan_frames"; }
236  GlobalVariable *createFrameSectionBound(Module &M, Type *Ty,
237  const char *Name) {
238  auto GV = new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
239  nullptr, Name);
240  GV->setVisibility(GlobalValue::HiddenVisibility);
241  return GV;
242  }
243 
244  /// This struct defines the shadow mapping using the rule:
245  /// shadow = (mem >> Scale) + Offset.
246  /// If InGlobal is true, then
247  /// extern char __hwasan_shadow[];
248  /// shadow = (mem >> Scale) + &__hwasan_shadow
249  /// If InTls is true, then
250  /// extern char *__hwasan_tls;
251  /// shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
252  struct ShadowMapping {
253  int Scale;
254  uint64_t Offset;
255  bool InGlobal;
256  bool InTls;
257 
258  void init(Triple &TargetTriple);
259  unsigned getAllocaAlignment() const { return 1U << Scale; }
260  };
261  ShadowMapping Mapping;
262 
263  Type *IntptrTy;
264  Type *Int8PtrTy;
265  Type *Int8Ty;
266  Type *Int32Ty;
267 
268  bool CompileKernel;
269  bool Recover;
270 
271  Function *HwasanCtorFunction;
272 
273  FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
274  FunctionCallee HwasanMemoryAccessCallbackSized[2];
275 
276  FunctionCallee HwasanTagMemoryFunc;
277  FunctionCallee HwasanGenerateTagFunc;
278  FunctionCallee HwasanThreadEnterFunc;
279 
280  Constant *ShadowGlobal;
281 
282  Value *LocalDynamicShadow = nullptr;
283  GlobalValue *ThreadPtrGlobal = nullptr;
284 };
285 
286 } // end anonymous namespace
287 
288 char HWAddressSanitizer::ID = 0;
289 
291  HWAddressSanitizer, "hwasan",
292  "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
293  false)
295  HWAddressSanitizer, "hwasan",
296  "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
297  false)
298 
300  bool Recover) {
301  assert(!CompileKernel || Recover);
302  return new HWAddressSanitizer(CompileKernel, Recover);
303 }
304 
305 /// Module-level initialization.
306 ///
307 /// inserts a call to __hwasan_init to the module's constructor list.
308 bool HWAddressSanitizer::doInitialization(Module &M) {
309  LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
310  auto &DL = M.getDataLayout();
311 
312  TargetTriple = Triple(M.getTargetTriple());
313 
314  Mapping.init(TargetTriple);
315 
316  C = &(M.getContext());
317  CurModuleUniqueId = getUniqueModuleId(&M);
318  IRBuilder<> IRB(*C);
319  IntptrTy = IRB.getIntPtrTy(DL);
320  Int8PtrTy = IRB.getInt8PtrTy();
321  Int8Ty = IRB.getInt8Ty();
322  Int32Ty = IRB.getInt32Ty();
323 
324  HwasanCtorFunction = nullptr;
325  if (!CompileKernel) {
326  std::tie(HwasanCtorFunction, std::ignore) =
327  createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName,
328  kHwasanInitName,
329  /*InitArgTypes=*/{},
330  /*InitArgs=*/{});
331  Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
332  HwasanCtorFunction->setComdat(CtorComdat);
333  appendToGlobalCtors(M, HwasanCtorFunction, 0, HwasanCtorFunction);
334 
335  // Create a zero-length global in __hwasan_frame so that the linker will
336  // always create start and stop symbols.
337  //
338  // N.B. If we ever start creating associated metadata in this pass this
339  // global will need to be associated with the ctor.
340  Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
341  auto GV =
342  new GlobalVariable(M, Int8Arr0Ty, /*isConstantGlobal*/ true,
344  Constant::getNullValue(Int8Arr0Ty), "__hwasan");
345  GV->setSection(getFrameSection());
346  GV->setComdat(CtorComdat);
347  appendToCompilerUsed(M, GV);
348 
349  IRBuilder<> IRBCtor(HwasanCtorFunction->getEntryBlock().getTerminator());
350  IRBCtor.CreateCall(
351  declareSanitizerInitFunction(M, "__hwasan_init_frames",
352  {Int8PtrTy, Int8PtrTy}),
353  {createFrameSectionBound(M, Int8Ty, getFrameSectionBeg()),
354  createFrameSectionBound(M, Int8Ty, getFrameSectionEnd())});
355  }
356 
357  if (!TargetTriple.isAndroid())
359  M, ThreadPtrGlobal = new GlobalVariable(
360  M, IntptrTy, false, GlobalVariable::ExternalLinkage, nullptr,
361  "__hwasan_tls", nullptr, GlobalVariable::InitialExecTLSModel));
362 
363  return true;
364 }
365 
366 void HWAddressSanitizer::initializeCallbacks(Module &M) {
367  IRBuilder<> IRB(*C);
368  for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
369  const std::string TypeStr = AccessIsWrite ? "store" : "load";
370  const std::string EndingStr = Recover ? "_noabort" : "";
371 
372  HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
373  ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
374  FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
375 
376  for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
377  AccessSizeIndex++) {
378  HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
380  ClMemoryAccessCallbackPrefix + TypeStr +
381  itostr(1ULL << AccessSizeIndex) + EndingStr,
382  FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
383  }
384  }
385 
386  HwasanTagMemoryFunc = M.getOrInsertFunction(
387  "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
388  HwasanGenerateTagFunc =
389  M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
390 
391  ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
392  ArrayType::get(IRB.getInt8Ty(), 0));
393 
394  const std::string MemIntrinCallbackPrefix =
395  CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
396  HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
397  IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
398  IRB.getInt8PtrTy(), IntptrTy);
399  HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
400  IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
401  IRB.getInt8PtrTy(), IntptrTy);
402  HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
403  IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
404  IRB.getInt32Ty(), IntptrTy);
405 
406  HwasanThreadEnterFunc =
407  M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy());
408 }
409 
410 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
411  // An empty inline asm with input reg == output reg.
412  // An opaque no-op cast, basically.
414  FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
415  StringRef(""), StringRef("=r,0"),
416  /*hasSideEffects=*/false);
417  return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
418 }
419 
420 Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
421  // Generate code only when dynamic addressing is needed.
422  if (Mapping.Offset != kDynamicShadowSentinel)
423  return nullptr;
424 
425  if (Mapping.InGlobal) {
426  return getDynamicShadowIfunc(IRB);
427  } else {
428  Value *GlobalDynamicAddress =
430  kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
431  return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
432  }
433 }
434 
435 Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I,
436  bool *IsWrite,
437  uint64_t *TypeSize,
438  unsigned *Alignment,
439  Value **MaybeMask) {
440  // Skip memory accesses inserted by another instrumentation.
441  if (I->getMetadata("nosanitize")) return nullptr;
442 
443  // Do not instrument the load fetching the dynamic shadow address.
444  if (LocalDynamicShadow == I)
445  return nullptr;
446 
447  Value *PtrOperand = nullptr;
448  const DataLayout &DL = I->getModule()->getDataLayout();
449  if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
450  if (!ClInstrumentReads) return nullptr;
451  *IsWrite = false;
452  *TypeSize = DL.getTypeStoreSizeInBits(LI->getType());
453  *Alignment = LI->getAlignment();
454  PtrOperand = LI->getPointerOperand();
455  } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
456  if (!ClInstrumentWrites) return nullptr;
457  *IsWrite = true;
458  *TypeSize = DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType());
459  *Alignment = SI->getAlignment();
460  PtrOperand = SI->getPointerOperand();
461  } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
462  if (!ClInstrumentAtomics) return nullptr;
463  *IsWrite = true;
464  *TypeSize = DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType());
465  *Alignment = 0;
466  PtrOperand = RMW->getPointerOperand();
467  } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
468  if (!ClInstrumentAtomics) return nullptr;
469  *IsWrite = true;
470  *TypeSize = DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
471  *Alignment = 0;
472  PtrOperand = XCHG->getPointerOperand();
473  }
474 
475  if (PtrOperand) {
476  // Do not instrument accesses from different address spaces; we cannot deal
477  // with them.
478  Type *PtrTy = cast<PointerType>(PtrOperand->getType()->getScalarType());
479  if (PtrTy->getPointerAddressSpace() != 0)
480  return nullptr;
481 
482  // Ignore swifterror addresses.
483  // swifterror memory addresses are mem2reg promoted by instruction
484  // selection. As such they cannot have regular uses like an instrumentation
485  // function and it makes no sense to track them as memory.
486  if (PtrOperand->isSwiftError())
487  return nullptr;
488  }
489 
490  return PtrOperand;
491 }
492 
493 static unsigned getPointerOperandIndex(Instruction *I) {
494  if (LoadInst *LI = dyn_cast<LoadInst>(I))
495  return LI->getPointerOperandIndex();
496  if (StoreInst *SI = dyn_cast<StoreInst>(I))
497  return SI->getPointerOperandIndex();
498  if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
499  return RMW->getPointerOperandIndex();
500  if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
501  return XCHG->getPointerOperandIndex();
502  report_fatal_error("Unexpected instruction");
503  return -1;
504 }
505 
506 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
507  size_t Res = countTrailingZeros(TypeSize / 8);
509  return Res;
510 }
511 
512 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
513  if (TargetTriple.isAArch64())
514  return;
515 
516  IRBuilder<> IRB(I);
517  Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
518  Value *UntaggedPtr =
519  IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
520  I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
521 }
522 
523 Value *HWAddressSanitizer::shadowBase() {
524  if (LocalDynamicShadow)
525  return LocalDynamicShadow;
526  return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, Mapping.Offset),
527  Int8PtrTy);
528 }
529 
530 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
531  // Mem >> Scale
532  Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
533  if (Mapping.Offset == 0)
534  return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
535  // (Mem >> Scale) + Offset
536  return IRB.CreateGEP(Int8Ty, shadowBase(), Shadow);
537 }
538 
539 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
540  unsigned AccessSizeIndex,
541  Instruction *InsertBefore) {
542  const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex;
543  IRBuilder<> IRB(InsertBefore);
544 
545  if (!ClInlineAllChecks && TargetTriple.isAArch64() &&
546  TargetTriple.isOSBinFormatELF() && !Recover) {
547  Module *M = IRB.GetInsertBlock()->getParent()->getParent();
548  Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
549  IRB.CreateCall(
550  Intrinsic::getDeclaration(M, Intrinsic::hwasan_check_memaccess),
551  {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
552  return;
553  }
554 
555  Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
556  Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
557  IRB.getInt8Ty());
558  Value *AddrLong = untagPointer(IRB, PtrLong);
559  Value *Shadow = memToShadow(AddrLong, IRB);
560  Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
561  Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
562 
563  int matchAllTag = ClMatchAllTag.getNumOccurrences() > 0 ?
564  ClMatchAllTag : (CompileKernel ? 0xFF : -1);
565  if (matchAllTag != -1) {
566  Value *TagNotIgnored = IRB.CreateICmpNE(PtrTag,
567  ConstantInt::get(PtrTag->getType(), matchAllTag));
568  TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
569  }
570 
571  Instruction *CheckTerm =
572  SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, !Recover,
573  MDBuilder(*C).createBranchWeights(1, 100000));
574 
575  IRB.SetInsertPoint(CheckTerm);
576  InlineAsm *Asm;
577  switch (TargetTriple.getArch()) {
578  case Triple::x86_64:
579  // The signal handler will find the data address in rdi.
580  Asm = InlineAsm::get(
581  FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
582  "int3\nnopl " + itostr(0x40 + AccessInfo) + "(%rax)",
583  "{rdi}",
584  /*hasSideEffects=*/true);
585  break;
586  case Triple::aarch64:
587  case Triple::aarch64_be:
588  // The signal handler will find the data address in x0.
589  Asm = InlineAsm::get(
590  FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
591  "brk #" + itostr(0x900 + AccessInfo),
592  "{x0}",
593  /*hasSideEffects=*/true);
594  break;
595  default:
596  report_fatal_error("unsupported architecture");
597  }
598  IRB.CreateCall(Asm, PtrLong);
599 }
600 
601 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
602  IRBuilder<> IRB(MI);
603  if (isa<MemTransferInst>(MI)) {
604  IRB.CreateCall(
605  isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
606  {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
607  IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
608  IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
609  } else if (isa<MemSetInst>(MI)) {
610  IRB.CreateCall(
611  HWAsanMemset,
612  {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
613  IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
614  IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
615  }
616  MI->eraseFromParent();
617 }
618 
619 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
620  LLVM_DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
621  bool IsWrite = false;
622  unsigned Alignment = 0;
623  uint64_t TypeSize = 0;
624  Value *MaybeMask = nullptr;
625 
626  if (ClInstrumentMemIntrinsics && isa<MemIntrinsic>(I)) {
627  instrumentMemIntrinsic(cast<MemIntrinsic>(I));
628  return true;
629  }
630 
631  Value *Addr =
632  isInterestingMemoryAccess(I, &IsWrite, &TypeSize, &Alignment, &MaybeMask);
633 
634  if (!Addr)
635  return false;
636 
637  if (MaybeMask)
638  return false; //FIXME
639 
640  IRBuilder<> IRB(I);
641  if (isPowerOf2_64(TypeSize) &&
642  (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
643  (Alignment >= (1UL << Mapping.Scale) || Alignment == 0 ||
644  Alignment >= TypeSize / 8)) {
645  size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
646  if (ClInstrumentWithCalls) {
647  IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex],
648  IRB.CreatePointerCast(Addr, IntptrTy));
649  } else {
650  instrumentMemAccessInline(Addr, IsWrite, AccessSizeIndex, I);
651  }
652  } else {
653  IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite],
654  {IRB.CreatePointerCast(Addr, IntptrTy),
655  ConstantInt::get(IntptrTy, TypeSize / 8)});
656  }
657  untagPointerOperand(I, Addr);
658 
659  return true;
660 }
661 
662 static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
663  uint64_t ArraySize = 1;
664  if (AI.isArrayAllocation()) {
665  const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize());
666  assert(CI && "non-constant array size");
667  ArraySize = CI->getZExtValue();
668  }
669  Type *Ty = AI.getAllocatedType();
670  uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
671  return SizeInBytes * ArraySize;
672 }
673 
674 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
675  Value *Tag) {
676  size_t Size = (getAllocaSizeInBytes(*AI) + Mapping.getAllocaAlignment() - 1) &
677  ~(Mapping.getAllocaAlignment() - 1);
678 
679  Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
680  if (ClInstrumentWithCalls) {
681  IRB.CreateCall(HwasanTagMemoryFunc,
682  {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
683  ConstantInt::get(IntptrTy, Size)});
684  } else {
685  size_t ShadowSize = Size >> Mapping.Scale;
686  Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
687  // If this memset is not inlined, it will be intercepted in the hwasan
688  // runtime library. That's OK, because the interceptor skips the checks if
689  // the address is in the shadow region.
690  // FIXME: the interceptor is not as fast as real memset. Consider lowering
691  // llvm.memset right here into either a sequence of stores, or a call to
692  // hwasan_tag_memory.
693  IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1);
694  }
695  return true;
696 }
697 
698 static unsigned RetagMask(unsigned AllocaNo) {
699  // A list of 8-bit numbers that have at most one run of non-zero bits.
700  // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
701  // masks.
702  // The list does not include the value 255, which is used for UAR.
703  static unsigned FastMasks[] = {
704  0, 1, 2, 3, 4, 6, 7, 8, 12, 14, 15, 16, 24,
705  28, 30, 31, 32, 48, 56, 60, 62, 63, 64, 96, 112, 120,
706  124, 126, 127, 128, 192, 224, 240, 248, 252, 254};
707  return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))];
708 }
709 
710 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
711  return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
712 }
713 
714 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
716  return getNextTagWithCall(IRB);
717  // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
718  // first).
719  Module *M = IRB.GetInsertBlock()->getParent()->getParent();
720  auto GetStackPointerFn =
721  Intrinsic::getDeclaration(M, Intrinsic::frameaddress);
722  Value *StackPointer = IRB.CreateCall(
723  GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())});
724 
725  // Extract some entropy from the stack pointer for the tags.
726  // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
727  // between functions).
728  Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy);
729  Value *StackTag =
730  IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20),
731  "hwasan.stack.base.tag");
732  return StackTag;
733 }
734 
735 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
736  AllocaInst *AI, unsigned AllocaNo) {
738  return getNextTagWithCall(IRB);
739  return IRB.CreateXor(StackTag,
740  ConstantInt::get(IntptrTy, RetagMask(AllocaNo)));
741 }
742 
743 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
744  if (ClUARRetagToZero)
745  return ConstantInt::get(IntptrTy, 0);
747  return getNextTagWithCall(IRB);
748  return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
749 }
750 
751 // Add a tag to an address.
752 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
753  Value *PtrLong, Value *Tag) {
754  Value *TaggedPtrLong;
755  if (CompileKernel) {
756  // Kernel addresses have 0xFF in the most significant byte.
757  Value *ShiftedTag = IRB.CreateOr(
758  IRB.CreateShl(Tag, kPointerTagShift),
759  ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
760  TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
761  } else {
762  // Userspace can simply do OR (tag << 56);
763  Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
764  TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
765  }
766  return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
767 }
768 
769 // Remove tag from an address.
770 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
771  Value *UntaggedPtrLong;
772  if (CompileKernel) {
773  // Kernel addresses have 0xFF in the most significant byte.
774  UntaggedPtrLong = IRB.CreateOr(PtrLong,
775  ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
776  } else {
777  // Userspace addresses have 0x00.
778  UntaggedPtrLong = IRB.CreateAnd(PtrLong,
779  ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
780  }
781  return UntaggedPtrLong;
782 }
783 
784 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
785  Module *M = IRB.GetInsertBlock()->getParent()->getParent();
786  if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
787  // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
788  // in Bionic's libc/private/bionic_tls.h.
789  Function *ThreadPointerFunc =
790  Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
791  Value *SlotPtr = IRB.CreatePointerCast(
792  IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
793  IRB.CreateCall(ThreadPointerFunc), 0x30),
794  Ty->getPointerTo(0));
795  return SlotPtr;
796  }
797  if (ThreadPtrGlobal)
798  return ThreadPtrGlobal;
799 
800 
801  return nullptr;
802 }
803 
804 // Creates a string with a description of the stack frame (set of Allocas).
805 // The string is intended to be human readable.
806 // The current form is: Size1 Name1; Size2 Name2; ...
807 std::string
808 HWAddressSanitizer::createFrameString(ArrayRef<AllocaInst *> Allocas) {
809  std::ostringstream Descr;
810  for (auto AI : Allocas)
811  Descr << getAllocaSizeInBytes(*AI) << " " << AI->getName().str() << "; ";
812  return Descr.str();
813 }
814 
815 // Creates a global in the frame section which consists of two pointers:
816 // the function PC and the frame string constant.
817 void HWAddressSanitizer::createFrameGlobal(Function &F,
818  const std::string &FrameString) {
819  Module &M = *F.getParent();
820  auto DescrGV = createPrivateGlobalForString(M, FrameString, true);
821  auto PtrPairTy = StructType::get(F.getType(), DescrGV->getType());
822  auto GV = new GlobalVariable(
823  M, PtrPairTy, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
824  ConstantStruct::get(PtrPairTy, (Constant *)&F, (Constant *)DescrGV),
825  "__hwasan");
826  GV->setSection(getFrameSection());
827  appendToCompilerUsed(M, GV);
828  // Put GV into the F's Comadat so that if F is deleted GV can be deleted too.
829  if (auto Comdat =
830  GetOrCreateFunctionComdat(F, TargetTriple, CurModuleUniqueId))
831  GV->setComdat(Comdat);
832 }
833 
834 Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB,
835  bool WithFrameRecord) {
836  if (!Mapping.InTls)
837  return getDynamicShadowNonTls(IRB);
838 
839  if (ClAllowIfunc && !WithFrameRecord && TargetTriple.isAndroid())
840  return getDynamicShadowIfunc(IRB);
841 
842  Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
843  assert(SlotPtr);
844 
845  Instruction *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
846 
847  Function *F = IRB.GetInsertBlock()->getParent();
848  if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") {
849  Value *ThreadLongEqZero =
850  IRB.CreateICmpEQ(ThreadLong, ConstantInt::get(IntptrTy, 0));
851  auto *Br = cast<BranchInst>(SplitBlockAndInsertIfThen(
852  ThreadLongEqZero, cast<Instruction>(ThreadLongEqZero)->getNextNode(),
853  false, MDBuilder(*C).createBranchWeights(1, 100000)));
854 
855  IRB.SetInsertPoint(Br);
856  // FIXME: This should call a new runtime function with a custom calling
857  // convention to avoid needing to spill all arguments here.
858  IRB.CreateCall(HwasanThreadEnterFunc);
859  LoadInst *ReloadThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
860 
861  IRB.SetInsertPoint(&*Br->getSuccessor(0)->begin());
862  PHINode *ThreadLongPhi = IRB.CreatePHI(IntptrTy, 2);
863  ThreadLongPhi->addIncoming(ThreadLong, ThreadLong->getParent());
864  ThreadLongPhi->addIncoming(ReloadThreadLong, ReloadThreadLong->getParent());
865  ThreadLong = ThreadLongPhi;
866  }
867 
868  // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
869  Value *ThreadLongMaybeUntagged =
870  TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong);
871 
872  if (WithFrameRecord) {
873  // Prepare ring buffer data.
874  auto PC = IRB.CreatePtrToInt(F, IntptrTy);
875  auto GetStackPointerFn =
876  Intrinsic::getDeclaration(F->getParent(), Intrinsic::frameaddress);
877  Value *SP = IRB.CreatePtrToInt(
878  IRB.CreateCall(GetStackPointerFn,
879  {Constant::getNullValue(IRB.getInt32Ty())}),
880  IntptrTy);
881  // Mix SP and PC. TODO: also add the tag to the mix.
882  // Assumptions:
883  // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero)
884  // SP is 0xsssssssssssSSSS0 (4 lower bits are zero)
885  // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
886  // 0xSSSSPPPPPPPPPPPP
887  SP = IRB.CreateShl(SP, 44);
888 
889  // Store data to ring buffer.
890  Value *RecordPtr =
891  IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0));
892  IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr);
893 
894  // Update the ring buffer. Top byte of ThreadLong defines the size of the
895  // buffer in pages, it must be a power of two, and the start of the buffer
896  // must be aligned by twice that much. Therefore wrap around of the ring
897  // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
898  // The use of AShr instead of LShr is due to
899  // https://bugs.llvm.org/show_bug.cgi?id=39030
900  // Runtime library makes sure not to use the highest bit.
901  Value *WrapMask = IRB.CreateXor(
902  IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
903  ConstantInt::get(IntptrTy, (uint64_t)-1));
904  Value *ThreadLongNew = IRB.CreateAnd(
905  IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
906  IRB.CreateStore(ThreadLongNew, SlotPtr);
907  }
908 
909  // Get shadow base address by aligning RecordPtr up.
910  // Note: this is not correct if the pointer is already aligned.
911  // Runtime library will make sure this never happens.
912  Value *ShadowBase = IRB.CreateAdd(
913  IRB.CreateOr(
914  ThreadLongMaybeUntagged,
915  ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
916  ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
917  ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
918  return ShadowBase;
919 }
920 
921 bool HWAddressSanitizer::instrumentStack(
923  SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) {
924  // Ideally, we want to calculate tagged stack base pointer, and rewrite all
925  // alloca addresses using that. Unfortunately, offsets are not known yet
926  // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
927  // temp, shift-OR it into each alloca address and xor with the retag mask.
928  // This generates one extra instruction per alloca use.
929  for (unsigned N = 0; N < Allocas.size(); ++N) {
930  auto *AI = Allocas[N];
931  IRBuilder<> IRB(AI->getNextNode());
932 
933  // Replace uses of the alloca with tagged address.
934  Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
935  Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
936  Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
937  std::string Name =
938  AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
939  Replacement->setName(Name + ".hwasan");
940 
941  for (auto UI = AI->use_begin(), UE = AI->use_end(); UI != UE;) {
942  Use &U = *UI++;
943  if (U.getUser() != AILong)
944  U.set(Replacement);
945  }
946 
947  tagAlloca(IRB, AI, Tag);
948 
949  for (auto RI : RetVec) {
950  IRB.SetInsertPoint(RI);
951 
952  // Re-tag alloca memory with the special UAR tag.
953  Value *Tag = getUARTag(IRB, StackTag);
954  tagAlloca(IRB, AI, Tag);
955  }
956  }
957 
958  return true;
959 }
960 
961 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
962  return (AI.getAllocatedType()->isSized() &&
963  // FIXME: instrument dynamic allocas, too
964  AI.isStaticAlloca() &&
965  // alloca() may be called with 0 size, ignore it.
966  getAllocaSizeInBytes(AI) > 0 &&
967  // We are only interested in allocas not promotable to registers.
968  // Promotable allocas are common under -O0.
969  !isAllocaPromotable(&AI) &&
970  // inalloca allocas are not treated as static, and we don't want
971  // dynamic alloca instrumentation for them as well.
972  !AI.isUsedWithInAlloca() &&
973  // swifterror allocas are register promoted by ISel
974  !AI.isSwiftError());
975 }
976 
978  if (&F == HwasanCtorFunction)
979  return false;
980 
981  if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
982  return false;
983 
984  LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
985 
986  SmallVector<Instruction*, 16> ToInstrument;
987  SmallVector<AllocaInst*, 8> AllocasToInstrument;
989  for (auto &BB : F) {
990  for (auto &Inst : BB) {
991  if (ClInstrumentStack)
992  if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
993  // Realign all allocas. We don't want small uninteresting allocas to
994  // hide in instrumented alloca's padding.
995  if (AI->getAlignment() < Mapping.getAllocaAlignment())
996  AI->setAlignment(Mapping.getAllocaAlignment());
997  // Instrument some of them.
998  if (isInterestingAlloca(*AI))
999  AllocasToInstrument.push_back(AI);
1000  continue;
1001  }
1002 
1003  if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
1004  isa<CleanupReturnInst>(Inst))
1005  RetVec.push_back(&Inst);
1006 
1007  Value *MaybeMask = nullptr;
1008  bool IsWrite;
1009  unsigned Alignment;
1010  uint64_t TypeSize;
1011  Value *Addr = isInterestingMemoryAccess(&Inst, &IsWrite, &TypeSize,
1012  &Alignment, &MaybeMask);
1013  if (Addr || isa<MemIntrinsic>(Inst))
1014  ToInstrument.push_back(&Inst);
1015  }
1016  }
1017 
1018  if (AllocasToInstrument.empty() && ToInstrument.empty())
1019  return false;
1020 
1021  if (ClCreateFrameDescriptions && !AllocasToInstrument.empty())
1022  createFrameGlobal(F, createFrameString(AllocasToInstrument));
1023 
1024  initializeCallbacks(*F.getParent());
1025 
1026  assert(!LocalDynamicShadow);
1027 
1028  Instruction *InsertPt = &*F.getEntryBlock().begin();
1029  IRBuilder<> EntryIRB(InsertPt);
1030  LocalDynamicShadow = emitPrologue(EntryIRB,
1031  /*WithFrameRecord*/ ClRecordStackHistory &&
1032  !AllocasToInstrument.empty());
1033 
1034  bool Changed = false;
1035  if (!AllocasToInstrument.empty()) {
1036  Value *StackTag =
1037  ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
1038  Changed |= instrumentStack(AllocasToInstrument, RetVec, StackTag);
1039  }
1040 
1041  // If we split the entry block, move any allocas that were originally in the
1042  // entry block back into the entry block so that they aren't treated as
1043  // dynamic allocas.
1044  if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1045  InsertPt = &*F.getEntryBlock().begin();
1046  for (auto II = EntryIRB.GetInsertBlock()->begin(),
1047  IE = EntryIRB.GetInsertBlock()->end();
1048  II != IE;) {
1049  Instruction *I = &*II++;
1050  if (auto *AI = dyn_cast<AllocaInst>(I))
1051  if (isa<ConstantInt>(AI->getArraySize()))
1052  I->moveBefore(InsertPt);
1053  }
1054  }
1055 
1056  for (auto Inst : ToInstrument)
1057  Changed |= instrumentMemAccess(Inst);
1058 
1059  LocalDynamicShadow = nullptr;
1060 
1061  return Changed;
1062 }
1063 
1064 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
1065  Scale = kDefaultShadowScale;
1066  if (ClMappingOffset.getNumOccurrences() > 0) {
1067  InGlobal = false;
1068  InTls = false;
1070  } else if (ClEnableKhwasan || ClInstrumentWithCalls) {
1071  InGlobal = false;
1072  InTls = false;
1073  Offset = 0;
1074  } else if (ClWithIfunc) {
1075  InGlobal = true;
1076  InTls = false;
1078  } else if (ClWithTls) {
1079  InGlobal = false;
1080  InTls = true;
1082  } else {
1083  InGlobal = false;
1084  InTls = false;
1086  }
1087 }
static cl::opt< bool > ClAllowIfunc("hwasan-allow-ifunc", cl::desc("allow the use of ifunc"), cl::Hidden, cl::init(false))
static cl::opt< int > ClMatchAllTag("hwasan-match-all-tag", cl::desc("don't report bad accesses via pointers with this tag"), cl::Hidden, cl::init(-1))
uint64_t CallInst * C
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue *> Values)
Adds global values to the llvm.compiler.used list.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:67
use_iterator use_end()
Definition: Value.h:346
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
uint64_t getTypeStoreSizeInBits(Type *Ty) const
Returns the maximum number of bits that may be overwritten by storing the specified type; always a mu...
Definition: DataLayout.h:426
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition: Module.h:239
Value * CreateConstGEP1_32(Value *Ptr, unsigned Idx0, const Twine &Name="")
Definition: IRBuilder.h:1551
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:218
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1878
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:288
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve &#39;CreateLoad(Ty, Ptr, "...")&#39; correctly, instead of converting the string to &#39;bool...
Definition: IRBuilder.h:1392
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1235
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:64
bool isSized(SmallPtrSetImpl< Type *> *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:264
void setAlignment(unsigned Align)
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
an instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:528
static cl::opt< bool > ClRecover("hwasan-recover", cl::desc("Enable recovery mode (continue-after-error)."), cl::Hidden, cl::init(false))
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:164
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1759
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:135
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Externally visible function.
Definition: GlobalValue.h:48
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:320
StringRef getName() const
Get a short "name" for the module.
Definition: Module.h:226
F(f)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:534
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, unsigned Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Definition: IRBuilder.h:403
An instruction for reading from memory.
Definition: Instructions.h:167
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:691
static const char *const kHwasanInitName
static unsigned getPointerOperandIndex(Instruction *I)
static const size_t kDefaultShadowScale
bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:264
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:268
static cl::opt< bool > ClInlineAllChecks("hwasan-inline-all-checks", cl::desc("inline all checks"), cl::Hidden, cl::init(false))
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:346
static const char *const kHwasanShadowMemoryDynamicAddress
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
unsigned getAlignment() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:112
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:96
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:243
static cl::opt< bool > ClInstrumentWrites("hwasan-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:651
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:742
IntegerType * getIntPtrTy(const DataLayout &DL, unsigned AddrSpace=0)
Fetch the type representing a pointer to an integer value.
Definition: IRBuilder.h:389
This file contains the simple types necessary to represent the attributes associated with functions a...
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1049
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:285
static StructType * get(LLVMContext &Context, ArrayRef< Type *> Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:341
Type * getVoidTy()
Fetch the type representing void.
Definition: IRBuilder.h:379
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1421
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1762
User * getUser() const LLVM_READONLY
Returns the User that contains this Use.
Definition: Use.cpp:40
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1767
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
std::string itostr(int64_t X)
Definition: StringExtras.h:238
bool isSwiftError() const
Return true if this value is a swifterror value.
Definition: Value.cpp:726
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:120
static const unsigned kShadowBaseAlignment
bool isUsedWithInAlloca() const
Return true if this alloca is used as an inalloca argument to a call.
Definition: Instructions.h:124
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:234
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
static const unsigned kPointerTagShift
An instruction for storing to memory.
Definition: Instructions.h:320
static cl::opt< bool > ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics", cl::desc("instrument memory intrinsics"), cl::Hidden, cl::init(true))
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1694
Function * getDeclaration(Module *M, ID id, ArrayRef< Type *> Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1018
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Definition: IRBuilder.h:126
Value * getOperand(unsigned i) const
Definition: User.h:169
static cl::opt< bool > ClInstrumentStack("hwasan-instrument-stack", cl::desc("instrument stack (allocas)"), cl::Hidden, cl::init(true))
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1217
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return &#39;this&#39;.
Definition: Type.h:303
static bool runOnFunction(Function &F, bool PostInlining)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:422
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:148
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1179
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:119
void set(Value *Val)
Definition: Value.h:670
bool hasName() const
Definition: Value.h:250
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
static cl::opt< std::string > ClMemoryAccessCallbackPrefix("hwasan-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__hwasan_"))
This is an important base class in LLVM.
Definition: Constant.h:41
INITIALIZE_PASS_BEGIN(HWAddressSanitizer, "hwasan", "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false) INITIALIZE_PASS_END(HWAddressSanitizer
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< bool > ClGenerateTagsWithCalls("hwasan-generate-tags-with-calls", cl::desc("generate new tags with runtime library calls"), cl::Hidden, cl::init(false))
static const size_t kNumberOfAccessSizes
static cl::opt< bool > ClInstrumentAtomics("hwasan-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:433
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
static FunctionType * get(Type *Result, ArrayRef< Type *> Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:296
static Constant * get(StructType *T, ArrayRef< Constant *> V)
Definition: Constants.cpp:1043
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1874
std::string getUniqueModuleId(Module *M)
Produce a unique identifier for this module by taking the MD5 sum of the names of the module&#39;s strong...
Comdat * getOrInsertComdat(StringRef Name)
Return the Comdat in the module with the specified name.
Definition: Module.cpp:482
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:92
size_t size() const
Definition: SmallVector.h:52
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Definition: IRBuilder.h:384
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
Definition: MDBuilder.cpp:37
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1690
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:105
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:43
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2004
Value * CreateGEP(Value *Ptr, ArrayRef< Value *> IdxList, const Twine &Name="")
Definition: IRBuilder.h:1493
This is the common base class for memset/memcpy/memmove.
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
iterator end()
Definition: BasicBlock.h:270
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:1836
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:839
Module.h This file contains the declarations for the Module class.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:621
GlobalVariable * createPrivateGlobalForString(Module &M, StringRef Str, bool AllowMerging, const char *NamePrefix="")
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:83
FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, AttributeList AttributeList)
Look up the specified function in the module symbol table.
Definition: Module.cpp:143
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
Comdat * GetOrCreateFunctionComdat(Function &F, Triple &T, const std::string &ModuleId)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:336
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1138
static size_t TypeSizeToSizeIndex(uint32_t TypeSize)
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1813
FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef< Type *> InitArgTypes)
static cl::opt< bool > ClWithTls("hwasan-with-tls", cl::desc("Access dynamic shadow through an thread-local pointer on " "platforms that support this"), cl::Hidden, cl::init(true))
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:435
static cl::opt< bool > ClEnableKhwasan("hwasan-kernel", cl::desc("Enable KernelHWAddressSanitizer instrumentation"), cl::Hidden, cl::init(false))
use_iterator use_begin()
Definition: Value.h:338
Constant * getOrInsertGlobal(StringRef Name, Type *Ty, function_ref< GlobalVariable *()> CreateGlobalCallback)
Look up the specified global in the module symbol table.
Definition: Module.cpp:204
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:194
static const uint64_t kDynamicShadowSentinel
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:106
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
static unsigned RetagMask(unsigned AllocaNo)
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:580
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:322
uint32_t Size
Definition: Profile.cpp:46
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2009
static cl::opt< bool > ClRecordStackHistory("hwasan-record-stack-history", cl::desc("Record stack frames with tagged allocations " "in a thread-local ring buffer"), cl::Hidden, cl::init(true))
static InlineAsm * get(FunctionType *Ty, StringRef AsmString, StringRef Constraints, bool hasSideEffects, bool isAlignStack=false, AsmDialect asmDialect=AD_ATT)
InlineAsm::get - Return the specified uniqued inline asm string.
Definition: InlineAsm.cpp:42
static cl::opt< bool > ClWithIfunc("hwasan-with-ifunc", cl::desc("Access dynamic shadow through an ifunc global on " "platforms that support this"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClInstrumentWithCalls("hwasan-instrument-with-calls", cl::desc("instrument reads and writes with callbacks"), cl::Hidden, cl::init(false))
FunctionPass * createHWAddressSanitizerPass(bool CompileKernel=false, bool Recover=false)
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1199
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1757
static cl::opt< bool > ClCreateFrameDescriptions("hwasan-create-frame-descriptions", cl::desc("create static frame descriptions"), cl::Hidden, cl::init(true))
static uint64_t getAllocaSizeInBytes(const AllocaInst &AI)
bool isArrayAllocation() const
Return true if there is an allocation size parameter to the allocation instruction that is not 1...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:565
LLVM Value Representation.
Definition: Value.h:72
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
Definition: Instruction.cpp:86
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:330
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1159
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
std::pair< Function *, FunctionCallee > createSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type *> InitArgTypes, ArrayRef< Value *> InitArgs, StringRef VersionCheckName=StringRef())
Creates sanitizer constructor function, and calls sanitizer&#39;s init function from it.
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size...
static cl::opt< unsigned long long > ClMappingOffset("hwasan-mapping-offset", cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"), cl::Hidden, cl::init(0))
static const char *const kHwasanModuleCtorName
#define LLVM_DEBUG(X)
Definition: Debug.h:122
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:273
IntegerType * Int32Ty
const BasicBlock * getParent() const
Definition: Instruction.h:66
an instruction to allocate memory on the stack
Definition: Instructions.h:59
static cl::opt< bool > ClUARRetagToZero("hwasan-uar-retag-to-zero", cl::desc("Clear alloca tags before returning from the function to allow " "non-instrumented and instrumented function calls mix. When set " "to false, allocas are retagged before returning from the " "function to detect use after return."), cl::Hidden, cl::init(true))
static cl::opt< bool > ClInstrumentReads("hwasan-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))