Bug Summary

File:llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
Warning:line 1220, column 52
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name MemorySanitizer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/Transforms/Instrumentation -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-12-09-002921-48462-1 -x c++ /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
1//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of MemorySanitizer, a detector of uninitialized
11/// reads.
12///
13/// The algorithm of the tool is similar to Memcheck
14/// (http://goo.gl/QKbem). We associate a few shadow bits with every
15/// byte of the application memory, poison the shadow of the malloc-ed
16/// or alloca-ed memory, load the shadow bits on every memory read,
17/// propagate the shadow bits through some of the arithmetic
18/// instruction (including MOV), store the shadow bits on every memory
19/// write, report a bug on some other instructions (e.g. JMP) if the
20/// associated shadow is poisoned.
21///
22/// But there are differences too. The first and the major one:
23/// compiler instrumentation instead of binary instrumentation. This
24/// gives us much better register allocation, possible compiler
25/// optimizations and a fast start-up. But this brings the major issue
26/// as well: msan needs to see all program events, including system
27/// calls and reads/writes in system libraries, so we either need to
28/// compile *everything* with msan or use a binary translation
29/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30/// Another difference from Memcheck is that we use 8 shadow bits per
31/// byte of application memory and use a direct shadow mapping. This
32/// greatly simplifies the instrumentation code and avoids races on
33/// shadow updates (Memcheck is single-threaded so races are not a
34/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35/// path storage that uses 8 bits per byte).
36///
37/// The default value of shadow is 0, which means "clean" (not poisoned).
38///
39/// Every module initializer should call __msan_init to ensure that the
40/// shadow memory is ready. On error, __msan_warning is called. Since
41/// parameters and return values may be passed via registers, we have a
42/// specialized thread-local shadow for return values
43/// (__msan_retval_tls) and parameters (__msan_param_tls).
44///
45/// Origin tracking.
46///
47/// MemorySanitizer can track origins (allocation points) of all uninitialized
48/// values. This behavior is controlled with a flag (msan-track-origins) and is
49/// disabled by default.
50///
51/// Origins are 4-byte values created and interpreted by the runtime library.
52/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53/// of application memory. Propagation of origins is basically a bunch of
54/// "select" instructions that pick the origin of a dirty argument, if an
55/// instruction has one.
56///
57/// Every 4 aligned, consecutive bytes of application memory have one origin
58/// value associated with them. If these bytes contain uninitialized data
59/// coming from 2 different allocations, the last store wins. Because of this,
60/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61/// practice.
62///
63/// Origins are meaningless for fully initialized values, so MemorySanitizer
64/// avoids storing origin to memory when a fully initialized value is stored.
65/// This way it avoids needless overwritting origin of the 4-byte region on
66/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67///
68/// Atomic handling.
69///
70/// Ideally, every atomic store of application value should update the
71/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72/// of two disjoint locations can not be done without severe slowdown.
73///
74/// Therefore, we implement an approximation that may err on the safe side.
75/// In this implementation, every atomically accessed location in the program
76/// may only change from (partially) uninitialized to fully initialized, but
77/// not the other way around. We load the shadow _after_ the application load,
78/// and we store the shadow _before_ the app store. Also, we always store clean
79/// shadow (if the application store is atomic). This way, if the store-load
80/// pair constitutes a happens-before arc, shadow store and load are correctly
81/// ordered such that the load will get either the value that was stored, or
82/// some later value (which is always clean).
83///
84/// This does not work very well with Compare-And-Swap (CAS) and
85/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86/// must store the new shadow before the app operation, and load the shadow
87/// after the app operation. Computers don't work this way. Current
88/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89/// value. It implements the store part as a simple atomic store by storing a
90/// clean shadow.
91///
92/// Instrumenting inline assembly.
93///
94/// For inline assembly code LLVM has little idea about which memory locations
95/// become initialized depending on the arguments. It can be possible to figure
96/// out which arguments are meant to point to inputs and outputs, but the
97/// actual semantics can be only visible at runtime. In the Linux kernel it's
98/// also possible that the arguments only indicate the offset for a base taken
99/// from a segment register, so it's dangerous to treat any asm() arguments as
100/// pointers. We take a conservative approach generating calls to
101/// __msan_instrument_asm_store(ptr, size)
102/// , which defer the memory unpoisoning to the runtime library.
103/// The latter can perform more complex address checks to figure out whether
104/// it's safe to touch the shadow memory.
105/// Like with atomic operations, we call __msan_instrument_asm_store() before
106/// the assembly call, so that changes to the shadow memory will be seen by
107/// other threads together with main memory initialization.
108///
109/// KernelMemorySanitizer (KMSAN) implementation.
110///
111/// The major differences between KMSAN and MSan instrumentation are:
112/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113/// - KMSAN allocates shadow and origin memory for each page separately, so
114/// there are no explicit accesses to shadow and origin in the
115/// instrumentation.
116/// Shadow and origin values for a particular X-byte memory location
117/// (X=1,2,4,8) are accessed through pointers obtained via the
118/// __msan_metadata_ptr_for_load_X(ptr)
119/// __msan_metadata_ptr_for_store_X(ptr)
120/// functions. The corresponding functions check that the X-byte accesses
121/// are possible and returns the pointers to shadow and origin memory.
122/// Arbitrary sized accesses are handled with:
123/// __msan_metadata_ptr_for_load_n(ptr, size)
124/// __msan_metadata_ptr_for_store_n(ptr, size);
125/// - TLS variables are stored in a single per-task struct. A call to a
126/// function __msan_get_context_state() returning a pointer to that struct
127/// is inserted into every instrumented function before the entry block;
128/// - __msan_warning() takes a 32-bit origin parameter;
129/// - local variables are poisoned with __msan_poison_alloca() upon function
130/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
131/// function;
132/// - the pass doesn't declare any global variables or add global constructors
133/// to the translation unit.
134///
135/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
136/// calls, making sure we're on the safe side wrt. possible false positives.
137///
138/// KernelMemorySanitizer only supports X86_64 at the moment.
139///
140//===----------------------------------------------------------------------===//
141
142#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
143#include "llvm/ADT/APInt.h"
144#include "llvm/ADT/ArrayRef.h"
145#include "llvm/ADT/DepthFirstIterator.h"
146#include "llvm/ADT/SmallSet.h"
147#include "llvm/ADT/SmallString.h"
148#include "llvm/ADT/SmallVector.h"
149#include "llvm/ADT/StringExtras.h"
150#include "llvm/ADT/StringRef.h"
151#include "llvm/ADT/Triple.h"
152#include "llvm/Analysis/TargetLibraryInfo.h"
153#include "llvm/IR/Argument.h"
154#include "llvm/IR/Attributes.h"
155#include "llvm/IR/BasicBlock.h"
156#include "llvm/IR/CallSite.h"
157#include "llvm/IR/CallingConv.h"
158#include "llvm/IR/Constant.h"
159#include "llvm/IR/Constants.h"
160#include "llvm/IR/DataLayout.h"
161#include "llvm/IR/DerivedTypes.h"
162#include "llvm/IR/Function.h"
163#include "llvm/IR/GlobalValue.h"
164#include "llvm/IR/GlobalVariable.h"
165#include "llvm/IR/IRBuilder.h"
166#include "llvm/IR/InlineAsm.h"
167#include "llvm/IR/InstVisitor.h"
168#include "llvm/IR/InstrTypes.h"
169#include "llvm/IR/Instruction.h"
170#include "llvm/IR/Instructions.h"
171#include "llvm/IR/IntrinsicInst.h"
172#include "llvm/IR/Intrinsics.h"
173#include "llvm/IR/LLVMContext.h"
174#include "llvm/IR/MDBuilder.h"
175#include "llvm/IR/Module.h"
176#include "llvm/IR/Type.h"
177#include "llvm/IR/Value.h"
178#include "llvm/IR/ValueMap.h"
179#include "llvm/Pass.h"
180#include "llvm/Support/AtomicOrdering.h"
181#include "llvm/Support/Casting.h"
182#include "llvm/Support/CommandLine.h"
183#include "llvm/Support/Compiler.h"
184#include "llvm/Support/Debug.h"
185#include "llvm/Support/ErrorHandling.h"
186#include "llvm/Support/MathExtras.h"
187#include "llvm/Support/raw_ostream.h"
188#include "llvm/Transforms/Instrumentation.h"
189#include "llvm/Transforms/Utils/BasicBlockUtils.h"
190#include "llvm/Transforms/Utils/Local.h"
191#include "llvm/Transforms/Utils/ModuleUtils.h"
192#include <algorithm>
193#include <cassert>
194#include <cstddef>
195#include <cstdint>
196#include <memory>
197#include <string>
198#include <tuple>
199
200using namespace llvm;
201
202#define DEBUG_TYPE"msan" "msan"
203
204static const unsigned kOriginSize = 4;
205static const unsigned kMinOriginAlignment = 4;
206static const unsigned kShadowTLSAlignment = 8;
207
208// These constants must be kept in sync with the ones in msan.h.
209static const unsigned kParamTLSSize = 800;
210static const unsigned kRetvalTLSSize = 800;
211
212// Accesses sizes are powers of two: 1, 2, 4, 8.
213static const size_t kNumberOfAccessSizes = 4;
214
215/// Track origins of uninitialized values.
216///
217/// Adds a section to MemorySanitizer report that points to the allocation
218/// (stack or heap) the uninitialized bits came from originally.
219static cl::opt<int> ClTrackOrigins("msan-track-origins",
220 cl::desc("Track origins (allocation sites) of poisoned memory"),
221 cl::Hidden, cl::init(0));
222
223static cl::opt<bool> ClKeepGoing("msan-keep-going",
224 cl::desc("keep going after reporting a UMR"),
225 cl::Hidden, cl::init(false));
226
227static cl::opt<bool> ClPoisonStack("msan-poison-stack",
228 cl::desc("poison uninitialized stack variables"),
229 cl::Hidden, cl::init(true));
230
231static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
232 cl::desc("poison uninitialized stack variables with a call"),
233 cl::Hidden, cl::init(false));
234
235static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
236 cl::desc("poison uninitialized stack variables with the given pattern"),
237 cl::Hidden, cl::init(0xff));
238
239static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
240 cl::desc("poison undef temps"),
241 cl::Hidden, cl::init(true));
242
243static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
244 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
245 cl::Hidden, cl::init(true));
246
247static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
248 cl::desc("exact handling of relational integer ICmp"),
249 cl::Hidden, cl::init(false));
250
251static cl::opt<bool> ClHandleLifetimeIntrinsics(
252 "msan-handle-lifetime-intrinsics",
253 cl::desc(
254 "when possible, poison scoped variables at the beginning of the scope "
255 "(slower, but more precise)"),
256 cl::Hidden, cl::init(true));
257
258// When compiling the Linux kernel, we sometimes see false positives related to
259// MSan being unable to understand that inline assembly calls may initialize
260// local variables.
261// This flag makes the compiler conservatively unpoison every memory location
262// passed into an assembly call. Note that this may cause false positives.
263// Because it's impossible to figure out the array sizes, we can only unpoison
264// the first sizeof(type) bytes for each type* pointer.
265// The instrumentation is only enabled in KMSAN builds, and only if
266// -msan-handle-asm-conservative is on. This is done because we may want to
267// quickly disable assembly instrumentation when it breaks.
268static cl::opt<bool> ClHandleAsmConservative(
269 "msan-handle-asm-conservative",
270 cl::desc("conservative handling of inline assembly"), cl::Hidden,
271 cl::init(true));
272
273// This flag controls whether we check the shadow of the address
274// operand of load or store. Such bugs are very rare, since load from
275// a garbage address typically results in SEGV, but still happen
276// (e.g. only lower bits of address are garbage, or the access happens
277// early at program startup where malloc-ed memory is more likely to
278// be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
279static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
280 cl::desc("report accesses through a pointer which has poisoned shadow"),
281 cl::Hidden, cl::init(true));
282
283static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
284 cl::desc("print out instructions with default strict semantics"),
285 cl::Hidden, cl::init(false));
286
287static cl::opt<int> ClInstrumentationWithCallThreshold(
288 "msan-instrumentation-with-call-threshold",
289 cl::desc(
290 "If the function being instrumented requires more than "
291 "this number of checks and origin stores, use callbacks instead of "
292 "inline checks (-1 means never use callbacks)."),
293 cl::Hidden, cl::init(3500));
294
295static cl::opt<bool>
296 ClEnableKmsan("msan-kernel",
297 cl::desc("Enable KernelMemorySanitizer instrumentation"),
298 cl::Hidden, cl::init(false));
299
300// This is an experiment to enable handling of cases where shadow is a non-zero
301// compile-time constant. For some unexplainable reason they were silently
302// ignored in the instrumentation.
303static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
304 cl::desc("Insert checks for constant shadow values"),
305 cl::Hidden, cl::init(false));
306
307// This is off by default because of a bug in gold:
308// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
309static cl::opt<bool> ClWithComdat("msan-with-comdat",
310 cl::desc("Place MSan constructors in comdat sections"),
311 cl::Hidden, cl::init(false));
312
313// These options allow to specify custom memory map parameters
314// See MemoryMapParams for details.
315static cl::opt<uint64_t> ClAndMask("msan-and-mask",
316 cl::desc("Define custom MSan AndMask"),
317 cl::Hidden, cl::init(0));
318
319static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
320 cl::desc("Define custom MSan XorMask"),
321 cl::Hidden, cl::init(0));
322
323static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
324 cl::desc("Define custom MSan ShadowBase"),
325 cl::Hidden, cl::init(0));
326
327static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
328 cl::desc("Define custom MSan OriginBase"),
329 cl::Hidden, cl::init(0));
330
331static const char *const kMsanModuleCtorName = "msan.module_ctor";
332static const char *const kMsanInitName = "__msan_init";
333
334namespace {
335
336// Memory map parameters used in application-to-shadow address calculation.
337// Offset = (Addr & ~AndMask) ^ XorMask
338// Shadow = ShadowBase + Offset
339// Origin = OriginBase + Offset
340struct MemoryMapParams {
341 uint64_t AndMask;
342 uint64_t XorMask;
343 uint64_t ShadowBase;
344 uint64_t OriginBase;
345};
346
347struct PlatformMemoryMapParams {
348 const MemoryMapParams *bits32;
349 const MemoryMapParams *bits64;
350};
351
352} // end anonymous namespace
353
354// i386 Linux
355static const MemoryMapParams Linux_I386_MemoryMapParams = {
356 0x000080000000, // AndMask
357 0, // XorMask (not used)
358 0, // ShadowBase (not used)
359 0x000040000000, // OriginBase
360};
361
362// x86_64 Linux
363static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
364#ifdef MSAN_LINUX_X86_64_OLD_MAPPING
365 0x400000000000, // AndMask
366 0, // XorMask (not used)
367 0, // ShadowBase (not used)
368 0x200000000000, // OriginBase
369#else
370 0, // AndMask (not used)
371 0x500000000000, // XorMask
372 0, // ShadowBase (not used)
373 0x100000000000, // OriginBase
374#endif
375};
376
377// mips64 Linux
378static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
379 0, // AndMask (not used)
380 0x008000000000, // XorMask
381 0, // ShadowBase (not used)
382 0x002000000000, // OriginBase
383};
384
385// ppc64 Linux
386static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
387 0xE00000000000, // AndMask
388 0x100000000000, // XorMask
389 0x080000000000, // ShadowBase
390 0x1C0000000000, // OriginBase
391};
392
393// aarch64 Linux
394static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
395 0, // AndMask (not used)
396 0x06000000000, // XorMask
397 0, // ShadowBase (not used)
398 0x01000000000, // OriginBase
399};
400
401// i386 FreeBSD
402static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
403 0x000180000000, // AndMask
404 0x000040000000, // XorMask
405 0x000020000000, // ShadowBase
406 0x000700000000, // OriginBase
407};
408
409// x86_64 FreeBSD
410static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
411 0xc00000000000, // AndMask
412 0x200000000000, // XorMask
413 0x100000000000, // ShadowBase
414 0x380000000000, // OriginBase
415};
416
417// x86_64 NetBSD
418static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
419 0, // AndMask
420 0x500000000000, // XorMask
421 0, // ShadowBase
422 0x100000000000, // OriginBase
423};
424
425static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
426 &Linux_I386_MemoryMapParams,
427 &Linux_X86_64_MemoryMapParams,
428};
429
430static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
431 nullptr,
432 &Linux_MIPS64_MemoryMapParams,
433};
434
435static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
436 nullptr,
437 &Linux_PowerPC64_MemoryMapParams,
438};
439
440static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
441 nullptr,
442 &Linux_AArch64_MemoryMapParams,
443};
444
445static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
446 &FreeBSD_I386_MemoryMapParams,
447 &FreeBSD_X86_64_MemoryMapParams,
448};
449
450static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
451 nullptr,
452 &NetBSD_X86_64_MemoryMapParams,
453};
454
455namespace {
456
457/// Instrument functions of a module to detect uninitialized reads.
458///
459/// Instantiating MemorySanitizer inserts the msan runtime library API function
460/// declarations into the module if they don't exist already. Instantiating
461/// ensures the __msan_init function is in the list of global constructors for
462/// the module.
463class MemorySanitizer {
464public:
465 MemorySanitizer(Module &M, MemorySanitizerOptions Options)
466 : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
467 Recover(Options.Recover) {
468 initializeModule(M);
469 }
470
471 // MSan cannot be moved or copied because of MapParams.
472 MemorySanitizer(MemorySanitizer &&) = delete;
473 MemorySanitizer &operator=(MemorySanitizer &&) = delete;
474 MemorySanitizer(const MemorySanitizer &) = delete;
475 MemorySanitizer &operator=(const MemorySanitizer &) = delete;
476
477 bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
478
479private:
480 friend struct MemorySanitizerVisitor;
481 friend struct VarArgAMD64Helper;
482 friend struct VarArgMIPS64Helper;
483 friend struct VarArgAArch64Helper;
484 friend struct VarArgPowerPC64Helper;
485
486 void initializeModule(Module &M);
487 void initializeCallbacks(Module &M);
488 void createKernelApi(Module &M);
489 void createUserspaceApi(Module &M);
490
491 /// True if we're compiling the Linux kernel.
492 bool CompileKernel;
493 /// Track origins (allocation points) of uninitialized values.
494 int TrackOrigins;
495 bool Recover;
496
497 LLVMContext *C;
498 Type *IntptrTy;
499 Type *OriginTy;
500
501 // XxxTLS variables represent the per-thread state in MSan and per-task state
502 // in KMSAN.
503 // For the userspace these point to thread-local globals. In the kernel land
504 // they point to the members of a per-task struct obtained via a call to
505 // __msan_get_context_state().
506
507 /// Thread-local shadow storage for function parameters.
508 Value *ParamTLS;
509
510 /// Thread-local origin storage for function parameters.
511 Value *ParamOriginTLS;
512
513 /// Thread-local shadow storage for function return value.
514 Value *RetvalTLS;
515
516 /// Thread-local origin storage for function return value.
517 Value *RetvalOriginTLS;
518
519 /// Thread-local shadow storage for in-register va_arg function
520 /// parameters (x86_64-specific).
521 Value *VAArgTLS;
522
523 /// Thread-local shadow storage for in-register va_arg function
524 /// parameters (x86_64-specific).
525 Value *VAArgOriginTLS;
526
527 /// Thread-local shadow storage for va_arg overflow area
528 /// (x86_64-specific).
529 Value *VAArgOverflowSizeTLS;
530
531 /// Thread-local space used to pass origin value to the UMR reporting
532 /// function.
533 Value *OriginTLS;
534
535 /// Are the instrumentation callbacks set up?
536 bool CallbacksInitialized = false;
537
538 /// The run-time callback to print a warning.
539 FunctionCallee WarningFn;
540
541 // These arrays are indexed by log2(AccessSize).
542 FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
543 FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
544
545 /// Run-time helper that generates a new origin value for a stack
546 /// allocation.
547 FunctionCallee MsanSetAllocaOrigin4Fn;
548
549 /// Run-time helper that poisons stack on function entry.
550 FunctionCallee MsanPoisonStackFn;
551
552 /// Run-time helper that records a store (or any event) of an
553 /// uninitialized value and returns an updated origin id encoding this info.
554 FunctionCallee MsanChainOriginFn;
555
556 /// MSan runtime replacements for memmove, memcpy and memset.
557 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
558
559 /// KMSAN callback for task-local function argument shadow.
560 StructType *MsanContextStateTy;
561 FunctionCallee MsanGetContextStateFn;
562
563 /// Functions for poisoning/unpoisoning local variables
564 FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
565
566 /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
567 /// pointers.
568 FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
569 FunctionCallee MsanMetadataPtrForLoad_1_8[4];
570 FunctionCallee MsanMetadataPtrForStore_1_8[4];
571 FunctionCallee MsanInstrumentAsmStoreFn;
572
573 /// Helper to choose between different MsanMetadataPtrXxx().
574 FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
575
576 /// Memory map parameters used in application-to-shadow calculation.
577 const MemoryMapParams *MapParams;
578
579 /// Custom memory map parameters used when -msan-shadow-base or
580 // -msan-origin-base is provided.
581 MemoryMapParams CustomMapParams;
582
583 MDNode *ColdCallWeights;
584
585 /// Branch weights for origin store.
586 MDNode *OriginStoreWeights;
587
588 /// An empty volatile inline asm that prevents callback merge.
589 InlineAsm *EmptyAsm;
590};
591
592void insertModuleCtor(Module &M) {
593 getOrCreateSanitizerCtorAndInitFunctions(
594 M, kMsanModuleCtorName, kMsanInitName,
595 /*InitArgTypes=*/{},
596 /*InitArgs=*/{},
597 // This callback is invoked when the functions are created the first
598 // time. Hook them into the global ctors list in that case:
599 [&](Function *Ctor, FunctionCallee) {
600 if (!ClWithComdat) {
601 appendToGlobalCtors(M, Ctor, 0);
602 return;
603 }
604 Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
605 Ctor->setComdat(MsanCtorComdat);
606 appendToGlobalCtors(M, Ctor, 0, Ctor);
607 });
608}
609
610/// A legacy function pass for msan instrumentation.
611///
612/// Instruments functions to detect unitialized reads.
613struct MemorySanitizerLegacyPass : public FunctionPass {
614 // Pass identification, replacement for typeid.
615 static char ID;
616
617 MemorySanitizerLegacyPass(MemorySanitizerOptions Options = {})
618 : FunctionPass(ID), Options(Options) {}
619 StringRef getPassName() const override { return "MemorySanitizerLegacyPass"; }
620
621 void getAnalysisUsage(AnalysisUsage &AU) const override {
622 AU.addRequired<TargetLibraryInfoWrapperPass>();
623 }
624
625 bool runOnFunction(Function &F) override {
626 return MSan->sanitizeFunction(
627 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F));
628 }
629 bool doInitialization(Module &M) override;
630
631 Optional<MemorySanitizer> MSan;
632 MemorySanitizerOptions Options;
633};
634
635template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
636 return (Opt.getNumOccurrences() > 0) ? Opt : Default;
637}
638
639} // end anonymous namespace
640
641MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K)
642 : Kernel(getOptOrDefault(ClEnableKmsan, K)),
643 TrackOrigins(getOptOrDefault(ClTrackOrigins, Kernel ? 2 : TO)),
644 Recover(getOptOrDefault(ClKeepGoing, Kernel || R)) {}
645
646PreservedAnalyses MemorySanitizerPass::run(Function &F,
647 FunctionAnalysisManager &FAM) {
648 MemorySanitizer Msan(*F.getParent(), Options);
649 if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
1
Calling 'MemorySanitizer::sanitizeFunction'
650 return PreservedAnalyses::none();
651 return PreservedAnalyses::all();
652}
653
654PreservedAnalyses MemorySanitizerPass::run(Module &M,
655 ModuleAnalysisManager &AM) {
656 if (Options.Kernel)
657 return PreservedAnalyses::all();
658 insertModuleCtor(M);
659 return PreservedAnalyses::none();
660}
661
662char MemorySanitizerLegacyPass::ID = 0;
663
664INITIALIZE_PASS_BEGIN(MemorySanitizerLegacyPass, "msan",static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
665 "MemorySanitizer: detects uninitialized reads.", false,static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
666 false)static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
667INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
668INITIALIZE_PASS_END(MemorySanitizerLegacyPass, "msan",PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
669 "MemorySanitizer: detects uninitialized reads.", false,PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
670 false)PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
671
672FunctionPass *
673llvm::createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options) {
674 return new MemorySanitizerLegacyPass(Options);
675}
676
677/// Create a non-const global initialized with the given string.
678///
679/// Creates a writable global for Str so that we can pass it to the
680/// run-time lib. Runtime uses first 4 bytes of the string to store the
681/// frame ID, so the string needs to be mutable.
682static GlobalVariable *createPrivateNonConstGlobalForString(Module &M,
683 StringRef Str) {
684 Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
685 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
686 GlobalValue::PrivateLinkage, StrConst, "");
687}
688
689/// Create KMSAN API callbacks.
690void MemorySanitizer::createKernelApi(Module &M) {
691 IRBuilder<> IRB(*C);
692
693 // These will be initialized in insertKmsanPrologue().
694 RetvalTLS = nullptr;
695 RetvalOriginTLS = nullptr;
696 ParamTLS = nullptr;
697 ParamOriginTLS = nullptr;
698 VAArgTLS = nullptr;
699 VAArgOriginTLS = nullptr;
700 VAArgOverflowSizeTLS = nullptr;
701 // OriginTLS is unused in the kernel.
702 OriginTLS = nullptr;
703
704 // __msan_warning() in the kernel takes an origin.
705 WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(),
706 IRB.getInt32Ty());
707 // Requests the per-task context state (kmsan_context_state*) from the
708 // runtime library.
709 MsanContextStateTy = StructType::get(
710 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
711 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
712 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
713 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
714 IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
715 OriginTy);
716 MsanGetContextStateFn = M.getOrInsertFunction(
717 "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
718
719 Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
720 PointerType::get(IRB.getInt32Ty(), 0));
721
722 for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
723 std::string name_load =
724 "__msan_metadata_ptr_for_load_" + std::to_string(size);
725 std::string name_store =
726 "__msan_metadata_ptr_for_store_" + std::to_string(size);
727 MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
728 name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
729 MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
730 name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
731 }
732
733 MsanMetadataPtrForLoadN = M.getOrInsertFunction(
734 "__msan_metadata_ptr_for_load_n", RetTy,
735 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
736 MsanMetadataPtrForStoreN = M.getOrInsertFunction(
737 "__msan_metadata_ptr_for_store_n", RetTy,
738 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
739
740 // Functions for poisoning and unpoisoning memory.
741 MsanPoisonAllocaFn =
742 M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(),
743 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy());
744 MsanUnpoisonAllocaFn = M.getOrInsertFunction(
745 "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
746}
747
748static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
749 return M.getOrInsertGlobal(Name, Ty, [&] {
750 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
751 nullptr, Name, nullptr,
752 GlobalVariable::InitialExecTLSModel);
753 });
754}
755
756/// Insert declarations for userspace-specific functions and globals.
757void MemorySanitizer::createUserspaceApi(Module &M) {
758 IRBuilder<> IRB(*C);
759 // Create the callback.
760 // FIXME: this function should have "Cold" calling conv,
761 // which is not yet implemented.
762 StringRef WarningFnName = Recover ? "__msan_warning"
763 : "__msan_warning_noreturn";
764 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
765
766 // Create the global TLS variables.
767 RetvalTLS =
768 getOrInsertGlobal(M, "__msan_retval_tls",
769 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
770
771 RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy);
772
773 ParamTLS =
774 getOrInsertGlobal(M, "__msan_param_tls",
775 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
776
777 ParamOriginTLS =
778 getOrInsertGlobal(M, "__msan_param_origin_tls",
779 ArrayType::get(OriginTy, kParamTLSSize / 4));
780
781 VAArgTLS =
782 getOrInsertGlobal(M, "__msan_va_arg_tls",
783 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
784
785 VAArgOriginTLS =
786 getOrInsertGlobal(M, "__msan_va_arg_origin_tls",
787 ArrayType::get(OriginTy, kParamTLSSize / 4));
788
789 VAArgOverflowSizeTLS =
790 getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
791 OriginTLS = getOrInsertGlobal(M, "__msan_origin_tls", IRB.getInt32Ty());
792
793 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
794 AccessSizeIndex++) {
795 unsigned AccessSize = 1 << AccessSizeIndex;
796 std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
797 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
798 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
799 IRB.getInt32Ty());
800
801 FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
802 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
803 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
804 IRB.getInt8PtrTy(), IRB.getInt32Ty());
805 }
806
807 MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
808 "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
809 IRB.getInt8PtrTy(), IntptrTy);
810 MsanPoisonStackFn =
811 M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
812 IRB.getInt8PtrTy(), IntptrTy);
813}
814
815/// Insert extern declaration of runtime-provided functions and globals.
816void MemorySanitizer::initializeCallbacks(Module &M) {
817 // Only do this once.
818 if (CallbacksInitialized)
819 return;
820
821 IRBuilder<> IRB(*C);
822 // Initialize callbacks that are common for kernel and userspace
823 // instrumentation.
824 MsanChainOriginFn = M.getOrInsertFunction(
825 "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
826 MemmoveFn = M.getOrInsertFunction(
827 "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
828 IRB.getInt8PtrTy(), IntptrTy);
829 MemcpyFn = M.getOrInsertFunction(
830 "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
831 IntptrTy);
832 MemsetFn = M.getOrInsertFunction(
833 "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
834 IntptrTy);
835 // We insert an empty inline asm after __msan_report* to avoid callback merge.
836 EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
837 StringRef(""), StringRef(""),
838 /*hasSideEffects=*/true);
839
840 MsanInstrumentAsmStoreFn =
841 M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
842 PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
843
844 if (CompileKernel) {
845 createKernelApi(M);
846 } else {
847 createUserspaceApi(M);
848 }
849 CallbacksInitialized = true;
850}
851
852FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
853 int size) {
854 FunctionCallee *Fns =
855 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
856 switch (size) {
857 case 1:
858 return Fns[0];
859 case 2:
860 return Fns[1];
861 case 4:
862 return Fns[2];
863 case 8:
864 return Fns[3];
865 default:
866 return nullptr;
867 }
868}
869
870/// Module-level initialization.
871///
872/// inserts a call to __msan_init to the module's constructor list.
873void MemorySanitizer::initializeModule(Module &M) {
874 auto &DL = M.getDataLayout();
875
876 bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
877 bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
878 // Check the overrides first
879 if (ShadowPassed || OriginPassed) {
880 CustomMapParams.AndMask = ClAndMask;
881 CustomMapParams.XorMask = ClXorMask;
882 CustomMapParams.ShadowBase = ClShadowBase;
883 CustomMapParams.OriginBase = ClOriginBase;
884 MapParams = &CustomMapParams;
885 } else {
886 Triple TargetTriple(M.getTargetTriple());
887 switch (TargetTriple.getOS()) {
888 case Triple::FreeBSD:
889 switch (TargetTriple.getArch()) {
890 case Triple::x86_64:
891 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
892 break;
893 case Triple::x86:
894 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
895 break;
896 default:
897 report_fatal_error("unsupported architecture");
898 }
899 break;
900 case Triple::NetBSD:
901 switch (TargetTriple.getArch()) {
902 case Triple::x86_64:
903 MapParams = NetBSD_X86_MemoryMapParams.bits64;
904 break;
905 default:
906 report_fatal_error("unsupported architecture");
907 }
908 break;
909 case Triple::Linux:
910 switch (TargetTriple.getArch()) {
911 case Triple::x86_64:
912 MapParams = Linux_X86_MemoryMapParams.bits64;
913 break;
914 case Triple::x86:
915 MapParams = Linux_X86_MemoryMapParams.bits32;
916 break;
917 case Triple::mips64:
918 case Triple::mips64el:
919 MapParams = Linux_MIPS_MemoryMapParams.bits64;
920 break;
921 case Triple::ppc64:
922 case Triple::ppc64le:
923 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
924 break;
925 case Triple::aarch64:
926 case Triple::aarch64_be:
927 MapParams = Linux_ARM_MemoryMapParams.bits64;
928 break;
929 default:
930 report_fatal_error("unsupported architecture");
931 }
932 break;
933 default:
934 report_fatal_error("unsupported operating system");
935 }
936 }
937
938 C = &(M.getContext());
939 IRBuilder<> IRB(*C);
940 IntptrTy = IRB.getIntPtrTy(DL);
941 OriginTy = IRB.getInt32Ty();
942
943 ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
944 OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
945
946 if (!CompileKernel) {
947 if (TrackOrigins)
948 M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
949 return new GlobalVariable(
950 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
951 IRB.getInt32(TrackOrigins), "__msan_track_origins");
952 });
953
954 if (Recover)
955 M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
956 return new GlobalVariable(M, IRB.getInt32Ty(), true,
957 GlobalValue::WeakODRLinkage,
958 IRB.getInt32(Recover), "__msan_keep_going");
959 });
960}
961}
962
963bool MemorySanitizerLegacyPass::doInitialization(Module &M) {
964 if (!Options.Kernel)
965 insertModuleCtor(M);
966 MSan.emplace(M, Options);
967 return true;
968}
969
970namespace {
971
972/// A helper class that handles instrumentation of VarArg
973/// functions on a particular platform.
974///
975/// Implementations are expected to insert the instrumentation
976/// necessary to propagate argument shadow through VarArg function
977/// calls. Visit* methods are called during an InstVisitor pass over
978/// the function, and should avoid creating new basic blocks. A new
979/// instance of this class is created for each instrumented function.
980struct VarArgHelper {
981 virtual ~VarArgHelper() = default;
982
983 /// Visit a CallSite.
984 virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0;
985
986 /// Visit a va_start call.
987 virtual void visitVAStartInst(VAStartInst &I) = 0;
988
989 /// Visit a va_copy call.
990 virtual void visitVACopyInst(VACopyInst &I) = 0;
991
992 /// Finalize function instrumentation.
993 ///
994 /// This method is called after visiting all interesting (see above)
995 /// instructions in a function.
996 virtual void finalizeInstrumentation() = 0;
997};
998
999struct MemorySanitizerVisitor;
1000
1001} // end anonymous namespace
1002
1003static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1004 MemorySanitizerVisitor &Visitor);
1005
1006static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
1007 if (TypeSize <= 8) return 0;
1008 return Log2_32_Ceil((TypeSize + 7) / 8);
1009}
1010
1011namespace {
1012
1013/// This class does all the work for a given function. Store and Load
1014/// instructions store and load corresponding shadow and origin
1015/// values. Most instructions propagate shadow from arguments to their
1016/// return values. Certain instructions (most importantly, BranchInst)
1017/// test their argument shadow and print reports (with a runtime call) if it's
1018/// non-zero.
1019struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1020 Function &F;
1021 MemorySanitizer &MS;
1022 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1023 ValueMap<Value*, Value*> ShadowMap, OriginMap;
1024 std::unique_ptr<VarArgHelper> VAHelper;
1025 const TargetLibraryInfo *TLI;
1026 BasicBlock *ActualFnStart;
1027
1028 // The following flags disable parts of MSan instrumentation based on
1029 // blacklist contents and command-line options.
1030 bool InsertChecks;
1031 bool PropagateShadow;
1032 bool PoisonStack;
1033 bool PoisonUndef;
1034 bool CheckReturnValue;
1035
1036 struct ShadowOriginAndInsertPoint {
1037 Value *Shadow;
1038 Value *Origin;
1039 Instruction *OrigIns;
1040
1041 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1042 : Shadow(S), Origin(O), OrigIns(I) {}
1043 };
1044 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1045 bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics;
1046 SmallSet<AllocaInst *, 16> AllocaSet;
1047 SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1048 SmallVector<StoreInst *, 16> StoreList;
1049
1050 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1051 const TargetLibraryInfo &TLI)
1052 : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
1053 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
1054 InsertChecks = SanitizeFunction;
1055 PropagateShadow = SanitizeFunction;
1056 PoisonStack = SanitizeFunction && ClPoisonStack;
1057 PoisonUndef = SanitizeFunction && ClPoisonUndef;
1058 // FIXME: Consider using SpecialCaseList to specify a list of functions that
1059 // must always return fully initialized values. For now, we hardcode "main".
1060 CheckReturnValue = SanitizeFunction && (F.getName() == "main");
1061
1062 MS.initializeCallbacks(*F.getParent());
1063 if (MS.CompileKernel)
1064 ActualFnStart = insertKmsanPrologue(F);
1065 else
1066 ActualFnStart = &F.getEntryBlock();
1067
1068 LLVM_DEBUG(if (!InsertChecks) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
1069 << "MemorySanitizer is not inserting checks into '"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
1070 << F.getName() << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
;
1071 }
1072
1073 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1074 if (MS.TrackOrigins <= 1) return V;
1075 return IRB.CreateCall(MS.MsanChainOriginFn, V);
1076 }
1077
1078 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1079 const DataLayout &DL = F.getParent()->getDataLayout();
1080 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1081 if (IntptrSize == kOriginSize) return Origin;
1082 assert(IntptrSize == kOriginSize * 2)((IntptrSize == kOriginSize * 2) ? static_cast<void> (0
) : __assert_fail ("IntptrSize == kOriginSize * 2", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1082, __PRETTY_FUNCTION__))
;
1083 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
1084 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
1085 }
1086
1087 /// Fill memory range with the given origin value.
1088 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1089 unsigned Size, unsigned Alignment) {
1090 const DataLayout &DL = F.getParent()->getDataLayout();
1091 unsigned IntptrAlignment = DL.getABITypeAlignment(MS.IntptrTy);
1092 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1093 assert(IntptrAlignment >= kMinOriginAlignment)((IntptrAlignment >= kMinOriginAlignment) ? static_cast<
void> (0) : __assert_fail ("IntptrAlignment >= kMinOriginAlignment"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1093, __PRETTY_FUNCTION__))
;
1094 assert(IntptrSize >= kOriginSize)((IntptrSize >= kOriginSize) ? static_cast<void> (0)
: __assert_fail ("IntptrSize >= kOriginSize", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1094, __PRETTY_FUNCTION__))
;
1095
1096 unsigned Ofs = 0;
1097 unsigned CurrentAlignment = Alignment;
1098 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1099 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1100 Value *IntptrOriginPtr =
1101 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
1102 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1103 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
1104 : IntptrOriginPtr;
1105 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
1106 Ofs += IntptrSize / kOriginSize;
1107 CurrentAlignment = IntptrAlignment;
1108 }
1109 }
1110
1111 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1112 Value *GEP =
1113 i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1114 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
1115 CurrentAlignment = kMinOriginAlignment;
1116 }
1117 }
1118
1119 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1120 Value *OriginPtr, unsigned Alignment, bool AsCall) {
1121 const DataLayout &DL = F.getParent()->getDataLayout();
1122 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1123 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
1124 if (Shadow->getType()->isAggregateType()) {
1125 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1126 OriginAlignment);
1127 } else {
1128 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
1129 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
1130 if (ConstantShadow) {
1131 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
1132 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1133 OriginAlignment);
1134 return;
1135 }
1136
1137 unsigned TypeSizeInBits =
1138 DL.getTypeSizeInBits(ConvertedShadow->getType());
1139 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1140 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1141 FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1142 Value *ConvertedShadow2 = IRB.CreateZExt(
1143 ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1144 IRB.CreateCall(Fn, {ConvertedShadow2,
1145 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
1146 Origin});
1147 } else {
1148 Value *Cmp = IRB.CreateICmpNE(
1149 ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp");
1150 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1151 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
1152 IRBuilder<> IRBNew(CheckTerm);
1153 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1154 OriginAlignment);
1155 }
1156 }
1157 }
1158
1159 void materializeStores(bool InstrumentWithCalls) {
1160 for (StoreInst *SI : StoreList) {
1161 IRBuilder<> IRB(SI);
1162 Value *Val = SI->getValueOperand();
1163 Value *Addr = SI->getPointerOperand();
1164 Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1165 Value *ShadowPtr, *OriginPtr;
1166 Type *ShadowTy = Shadow->getType();
1167 unsigned Alignment = SI->getAlignment();
1168 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1169 std::tie(ShadowPtr, OriginPtr) =
1170 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1171
1172 StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
1173 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " STORE: " << *NewSI <<
"\n"; } } while (false)
;
1174 (void)NewSI;
1175
1176 if (SI->isAtomic())
1177 SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
1178
1179 if (MS.TrackOrigins && !SI->isAtomic())
1180 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1181 OriginAlignment, InstrumentWithCalls);
1182 }
1183 }
1184
1185 /// Helper function to insert a warning at IRB's current insert point.
1186 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1187 if (!Origin)
1188 Origin = (Value *)IRB.getInt32(0);
1189 if (MS.CompileKernel) {
1190 IRB.CreateCall(MS.WarningFn, Origin);
1191 } else {
1192 if (MS.TrackOrigins) {
1193 IRB.CreateStore(Origin, MS.OriginTLS);
1194 }
1195 IRB.CreateCall(MS.WarningFn, {});
1196 }
1197 IRB.CreateCall(MS.EmptyAsm, {});
1198 // FIXME: Insert UnreachableInst if !MS.Recover?
1199 // This may invalidate some of the following checks and needs to be done
1200 // at the very end.
1201 }
1202
1203 void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
1204 bool AsCall) {
1205 IRBuilder<> IRB(OrigIns);
1206 LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " SHAD0 : " << *Shadow <<
"\n"; } } while (false)
;
11
Assuming 'DebugFlag' is false
12
Loop condition is false. Exiting loop
1207 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
13
'ConvertedShadow' initialized here
1208 LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " SHAD1 : " << *ConvertedShadow
<< "\n"; } } while (false)
;
14
Assuming 'DebugFlag' is false
15
Loop condition is false. Exiting loop
1209
1210 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
16
Assuming null pointer is passed into cast
17
Assuming pointer value is null
1211 if (ConstantShadow
17.1
'ConstantShadow' is null
) {
18
Taking false branch
1212 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
1213 insertWarningFn(IRB, Origin);
1214 }
1215 return;
1216 }
1217
1218 const DataLayout &DL = OrigIns->getModule()->getDataLayout();
1219
1220 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
19
Called C++ object pointer is null
1221 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1222 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1223 FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1224 Value *ConvertedShadow2 =
1225 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1226 IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin
1227 ? Origin
1228 : (Value *)IRB.getInt32(0)});
1229 } else {
1230 Value *Cmp = IRB.CreateICmpNE(ConvertedShadow,
1231 getCleanShadow(ConvertedShadow), "_mscmp");
1232 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1233 Cmp, OrigIns,
1234 /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
1235
1236 IRB.SetInsertPoint(CheckTerm);
1237 insertWarningFn(IRB, Origin);
1238 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " CHECK: " << *Cmp <<
"\n"; } } while (false)
;
1239 }
1240 }
1241
1242 void materializeChecks(bool InstrumentWithCalls) {
1243 for (const auto &ShadowData : InstrumentationList) {
9
Assuming '__begin2' is not equal to '__end2'
1244 Instruction *OrigIns = ShadowData.OrigIns;
1245 Value *Shadow = ShadowData.Shadow;
1246 Value *Origin = ShadowData.Origin;
1247 materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
10
Calling 'MemorySanitizerVisitor::materializeOneCheck'
1248 }
1249 LLVM_DEBUG(dbgs() << "DONE:\n" << F)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "DONE:\n" << F; } } while (
false)
;
1250 }
1251
1252 BasicBlock *insertKmsanPrologue(Function &F) {
1253 BasicBlock *ret =
1254 SplitBlock(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHI());
1255 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
1256 Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
1257 Constant *Zero = IRB.getInt32(0);
1258 MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1259 {Zero, IRB.getInt32(0)}, "param_shadow");
1260 MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1261 {Zero, IRB.getInt32(1)}, "retval_shadow");
1262 MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1263 {Zero, IRB.getInt32(2)}, "va_arg_shadow");
1264 MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1265 {Zero, IRB.getInt32(3)}, "va_arg_origin");
1266 MS.VAArgOverflowSizeTLS =
1267 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1268 {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
1269 MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1270 {Zero, IRB.getInt32(5)}, "param_origin");
1271 MS.RetvalOriginTLS =
1272 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1273 {Zero, IRB.getInt32(6)}, "retval_origin");
1274 return ret;
1275 }
1276
1277 /// Add MemorySanitizer instrumentation to a function.
1278 bool runOnFunction() {
1279 // In the presence of unreachable blocks, we may see Phi nodes with
1280 // incoming nodes from such blocks. Since InstVisitor skips unreachable
1281 // blocks, such nodes will not have any shadow value associated with them.
1282 // It's easier to remove unreachable blocks than deal with missing shadow.
1283 removeUnreachableBlocks(F);
1284
1285 // Iterate all BBs in depth-first order and create shadow instructions
1286 // for all instructions (where applicable).
1287 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1288 for (BasicBlock *BB : depth_first(ActualFnStart))
1289 visit(*BB);
1290
1291 // Finalize PHI nodes.
1292 for (PHINode *PN : ShadowPHINodes) {
4
Assuming '__begin2' is equal to '__end2'
1293 PHINode *PNS = cast<PHINode>(getShadow(PN));
1294 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1295 size_t NumValues = PN->getNumIncomingValues();
1296 for (size_t v = 0; v < NumValues; v++) {
1297 PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1298 if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1299 }
1300 }
1301
1302 VAHelper->finalizeInstrumentation();
1303
1304 // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1305 // instrumenting only allocas.
1306 if (InstrumentLifetimeStart) {
5
Assuming field 'InstrumentLifetimeStart' is false
6
Taking false branch
1307 for (auto Item : LifetimeStartList) {
1308 instrumentAlloca(*Item.second, Item.first);
1309 AllocaSet.erase(Item.second);
1310 }
1311 }
1312 // Poison the allocas for which we didn't instrument the corresponding
1313 // lifetime intrinsics.
1314 for (AllocaInst *AI : AllocaSet)
1315 instrumentAlloca(*AI);
1316
1317 bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
7
Assuming the condition is false
1318 InstrumentationList.size() + StoreList.size() >
1319 (unsigned)ClInstrumentationWithCallThreshold;
1320
1321 // Insert shadow value checks.
1322 materializeChecks(InstrumentWithCalls);
8
Calling 'MemorySanitizerVisitor::materializeChecks'
1323
1324 // Delayed instrumentation of StoreInst.
1325 // This may not add new address checks.
1326 materializeStores(InstrumentWithCalls);
1327
1328 return true;
1329 }
1330
1331 /// Compute the shadow type that corresponds to a given Value.
1332 Type *getShadowTy(Value *V) {
1333 return getShadowTy(V->getType());
1334 }
1335
1336 /// Compute the shadow type that corresponds to a given Type.
1337 Type *getShadowTy(Type *OrigTy) {
1338 if (!OrigTy->isSized()) {
1339 return nullptr;
1340 }
1341 // For integer type, shadow is the same as the original type.
1342 // This may return weird-sized types like i1.
1343 if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
1344 return IT;
1345 const DataLayout &DL = F.getParent()->getDataLayout();
1346 if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1347 uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1348 return VectorType::get(IntegerType::get(*MS.C, EltSize),
1349 VT->getNumElements());
1350 }
1351 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1352 return ArrayType::get(getShadowTy(AT->getElementType()),
1353 AT->getNumElements());
1354 }
1355 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1356 SmallVector<Type*, 4> Elements;
1357 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1358 Elements.push_back(getShadowTy(ST->getElementType(i)));
1359 StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
1360 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "getShadowTy: " << *ST <<
" ===> " << *Res << "\n"; } } while (false)
;
1361 return Res;
1362 }
1363 uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
1364 return IntegerType::get(*MS.C, TypeSize);
1365 }
1366
1367 /// Flatten a vector type.
1368 Type *getShadowTyNoVec(Type *ty) {
1369 if (VectorType *vt = dyn_cast<VectorType>(ty))
1370 return IntegerType::get(*MS.C, vt->getBitWidth());
1371 return ty;
1372 }
1373
1374 /// Convert a shadow value to it's flattened variant.
1375 Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) {
1376 Type *Ty = V->getType();
1377 Type *NoVecTy = getShadowTyNoVec(Ty);
1378 if (Ty == NoVecTy) return V;
1379 return IRB.CreateBitCast(V, NoVecTy);
1380 }
1381
1382 /// Compute the integer shadow offset that corresponds to a given
1383 /// application address.
1384 ///
1385 /// Offset = (Addr & ~AndMask) ^ XorMask
1386 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1387 Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
1388
1389 uint64_t AndMask = MS.MapParams->AndMask;
1390 if (AndMask)
1391 OffsetLong =
1392 IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
1393
1394 uint64_t XorMask = MS.MapParams->XorMask;
1395 if (XorMask)
1396 OffsetLong =
1397 IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
1398 return OffsetLong;
1399 }
1400
1401 /// Compute the shadow and origin addresses corresponding to a given
1402 /// application address.
1403 ///
1404 /// Shadow = ShadowBase + Offset
1405 /// Origin = (OriginBase + Offset) & ~3ULL
1406 std::pair<Value *, Value *> getShadowOriginPtrUserspace(Value *Addr,
1407 IRBuilder<> &IRB,
1408 Type *ShadowTy,
1409 unsigned Alignment) {
1410 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1411 Value *ShadowLong = ShadowOffset;
1412 uint64_t ShadowBase = MS.MapParams->ShadowBase;
1413 if (ShadowBase != 0) {
1414 ShadowLong =
1415 IRB.CreateAdd(ShadowLong,
1416 ConstantInt::get(MS.IntptrTy, ShadowBase));
1417 }
1418 Value *ShadowPtr =
1419 IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1420 Value *OriginPtr = nullptr;
1421 if (MS.TrackOrigins) {
1422 Value *OriginLong = ShadowOffset;
1423 uint64_t OriginBase = MS.MapParams->OriginBase;
1424 if (OriginBase != 0)
1425 OriginLong = IRB.CreateAdd(OriginLong,
1426 ConstantInt::get(MS.IntptrTy, OriginBase));
1427 if (Alignment < kMinOriginAlignment) {
1428 uint64_t Mask = kMinOriginAlignment - 1;
1429 OriginLong =
1430 IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
1431 }
1432 OriginPtr =
1433 IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0));
1434 }
1435 return std::make_pair(ShadowPtr, OriginPtr);
1436 }
1437
1438 std::pair<Value *, Value *>
1439 getShadowOriginPtrKernel(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1440 unsigned Alignment, bool isStore) {
1441 Value *ShadowOriginPtrs;
1442 const DataLayout &DL = F.getParent()->getDataLayout();
1443 int Size = DL.getTypeStoreSize(ShadowTy);
1444
1445 FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
1446 Value *AddrCast =
1447 IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
1448 if (Getter) {
1449 ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
1450 } else {
1451 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
1452 ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN
1453 : MS.MsanMetadataPtrForLoadN,
1454 {AddrCast, SizeVal});
1455 }
1456 Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
1457 ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
1458 Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
1459
1460 return std::make_pair(ShadowPtr, OriginPtr);
1461 }
1462
1463 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1464 Type *ShadowTy,
1465 unsigned Alignment,
1466 bool isStore) {
1467 std::pair<Value *, Value *> ret;
1468 if (MS.CompileKernel)
1469 ret = getShadowOriginPtrKernel(Addr, IRB, ShadowTy, Alignment, isStore);
1470 else
1471 ret = getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1472 return ret;
1473 }
1474
1475 /// Compute the shadow address for a given function argument.
1476 ///
1477 /// Shadow = ParamTLS+ArgOffset.
1478 Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
1479 int ArgOffset) {
1480 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1481 if (ArgOffset)
1482 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1483 return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1484 "_msarg");
1485 }
1486
1487 /// Compute the origin address for a given function argument.
1488 Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1489 int ArgOffset) {
1490 if (!MS.TrackOrigins)
1491 return nullptr;
1492 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1493 if (ArgOffset)
1494 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1495 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1496 "_msarg_o");
1497 }
1498
1499 /// Compute the shadow address for a retval.
1500 Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1501 return IRB.CreatePointerCast(MS.RetvalTLS,
1502 PointerType::get(getShadowTy(A), 0),
1503 "_msret");
1504 }
1505
1506 /// Compute the origin address for a retval.
1507 Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1508 // We keep a single origin for the entire retval. Might be too optimistic.
1509 return MS.RetvalOriginTLS;
1510 }
1511
1512 /// Set SV to be the shadow value for V.
1513 void setShadow(Value *V, Value *SV) {
1514 assert(!ShadowMap.count(V) && "Values may only have one shadow")((!ShadowMap.count(V) && "Values may only have one shadow"
) ? static_cast<void> (0) : __assert_fail ("!ShadowMap.count(V) && \"Values may only have one shadow\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1514, __PRETTY_FUNCTION__))
;
1515 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1516 }
1517
1518 /// Set Origin to be the origin value for V.
1519 void setOrigin(Value *V, Value *Origin) {
1520 if (!MS.TrackOrigins) return;
1521 assert(!OriginMap.count(V) && "Values may only have one origin")((!OriginMap.count(V) && "Values may only have one origin"
) ? static_cast<void> (0) : __assert_fail ("!OriginMap.count(V) && \"Values may only have one origin\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1521, __PRETTY_FUNCTION__))
;
1522 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "ORIGIN: " << *V << " ==> "
<< *Origin << "\n"; } } while (false)
;
1523 OriginMap[V] = Origin;
1524 }
1525
1526 Constant *getCleanShadow(Type *OrigTy) {
1527 Type *ShadowTy = getShadowTy(OrigTy);
1528 if (!ShadowTy)
1529 return nullptr;
1530 return Constant::getNullValue(ShadowTy);
1531 }
1532
1533 /// Create a clean shadow value for a given value.
1534 ///
1535 /// Clean shadow (all zeroes) means all bits of the value are defined
1536 /// (initialized).
1537 Constant *getCleanShadow(Value *V) {
1538 return getCleanShadow(V->getType());
1539 }
1540
1541 /// Create a dirty shadow of a given shadow type.
1542 Constant *getPoisonedShadow(Type *ShadowTy) {
1543 assert(ShadowTy)((ShadowTy) ? static_cast<void> (0) : __assert_fail ("ShadowTy"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1543, __PRETTY_FUNCTION__))
;
1544 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1545 return Constant::getAllOnesValue(ShadowTy);
1546 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1547 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1548 getPoisonedShadow(AT->getElementType()));
1549 return ConstantArray::get(AT, Vals);
1550 }
1551 if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1552 SmallVector<Constant *, 4> Vals;
1553 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1554 Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1555 return ConstantStruct::get(ST, Vals);
1556 }
1557 llvm_unreachable("Unexpected shadow type")::llvm::llvm_unreachable_internal("Unexpected shadow type", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1557)
;
1558 }
1559
1560 /// Create a dirty shadow for a given value.
1561 Constant *getPoisonedShadow(Value *V) {
1562 Type *ShadowTy = getShadowTy(V);
1563 if (!ShadowTy)
1564 return nullptr;
1565 return getPoisonedShadow(ShadowTy);
1566 }
1567
1568 /// Create a clean (zero) origin.
1569 Value *getCleanOrigin() {
1570 return Constant::getNullValue(MS.OriginTy);
1571 }
1572
1573 /// Get the shadow value for a given Value.
1574 ///
1575 /// This function either returns the value set earlier with setShadow,
1576 /// or extracts if from ParamTLS (for function arguments).
1577 Value *getShadow(Value *V) {
1578 if (!PropagateShadow) return getCleanShadow(V);
1579 if (Instruction *I = dyn_cast<Instruction>(V)) {
1580 if (I->getMetadata("nosanitize"))
1581 return getCleanShadow(V);
1582 // For instructions the shadow is already stored in the map.
1583 Value *Shadow = ShadowMap[V];
1584 if (!Shadow) {
1585 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "No shadow: " << *V <<
"\n" << *(I->getParent()); } } while (false)
;
1586 (void)I;
1587 assert(Shadow && "No shadow for a value")((Shadow && "No shadow for a value") ? static_cast<
void> (0) : __assert_fail ("Shadow && \"No shadow for a value\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1587, __PRETTY_FUNCTION__))
;
1588 }
1589 return Shadow;
1590 }
1591 if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1592 Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V);
1593 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Undef: " << *U << " ==> "
<< *AllOnes << "\n"; } } while (false)
;
1594 (void)U;
1595 return AllOnes;
1596 }
1597 if (Argument *A = dyn_cast<Argument>(V)) {
1598 // For arguments we compute the shadow on demand and store it in the map.
1599 Value **ShadowPtr = &ShadowMap[V];
1600 if (*ShadowPtr)
1601 return *ShadowPtr;
1602 Function *F = A->getParent();
1603 IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI());
1604 unsigned ArgOffset = 0;
1605 const DataLayout &DL = F->getParent()->getDataLayout();
1606 for (auto &FArg : F->args()) {
1607 if (!FArg.getType()->isSized()) {
1608 LLVM_DEBUG(dbgs() << "Arg is not sized\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Arg is not sized\n"; } } while (
false)
;
1609 continue;
1610 }
1611 unsigned Size =
1612 FArg.hasByValAttr()
1613 ? DL.getTypeAllocSize(FArg.getType()->getPointerElementType())
1614 : DL.getTypeAllocSize(FArg.getType());
1615 if (A == &FArg) {
1616 bool Overflow = ArgOffset + Size > kParamTLSSize;
1617 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1618 if (FArg.hasByValAttr()) {
1619 // ByVal pointer itself has clean shadow. We copy the actual
1620 // argument shadow to the underlying memory.
1621 // Figure out maximal valid memcpy alignment.
1622 unsigned ArgAlign = FArg.getParamAlignment();
1623 if (ArgAlign == 0) {
1624 Type *EltType = A->getType()->getPointerElementType();
1625 ArgAlign = DL.getABITypeAlignment(EltType);
1626 }
1627 Value *CpShadowPtr =
1628 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1629 /*isStore*/ true)
1630 .first;
1631 // TODO(glider): need to copy origins.
1632 if (Overflow) {
1633 // ParamTLS overflow.
1634 EntryIRB.CreateMemSet(
1635 CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
1636 Size, ArgAlign);
1637 } else {
1638 unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1639 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
1640 CopyAlign, Size);
1641 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ByValCpy: " << *Cpy <<
"\n"; } } while (false)
;
1642 (void)Cpy;
1643 }
1644 *ShadowPtr = getCleanShadow(V);
1645 } else {
1646 if (Overflow) {
1647 // ParamTLS overflow.
1648 *ShadowPtr = getCleanShadow(V);
1649 } else {
1650 *ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base,
1651 kShadowTLSAlignment);
1652 }
1653 }
1654 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ARG: " << FArg <<
" ==> " << **ShadowPtr << "\n"; } } while (false
)
1655 << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ARG: " << FArg <<
" ==> " << **ShadowPtr << "\n"; } } while (false
)
;
1656 if (MS.TrackOrigins && !Overflow) {
1657 Value *OriginPtr =
1658 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1659 setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
1660 } else {
1661 setOrigin(A, getCleanOrigin());
1662 }
1663 }
1664 ArgOffset += alignTo(Size, kShadowTLSAlignment);
1665 }
1666 assert(*ShadowPtr && "Could not find shadow for an argument")((*ShadowPtr && "Could not find shadow for an argument"
) ? static_cast<void> (0) : __assert_fail ("*ShadowPtr && \"Could not find shadow for an argument\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1666, __PRETTY_FUNCTION__))
;
1667 return *ShadowPtr;
1668 }
1669 // For everything else the shadow is zero.
1670 return getCleanShadow(V);
1671 }
1672
1673 /// Get the shadow for i-th argument of the instruction I.
1674 Value *getShadow(Instruction *I, int i) {
1675 return getShadow(I->getOperand(i));
1676 }
1677
1678 /// Get the origin for a value.
1679 Value *getOrigin(Value *V) {
1680 if (!MS.TrackOrigins) return nullptr;
1681 if (!PropagateShadow) return getCleanOrigin();
1682 if (isa<Constant>(V)) return getCleanOrigin();
1683 assert((isa<Instruction>(V) || isa<Argument>(V)) &&(((isa<Instruction>(V) || isa<Argument>(V)) &&
"Unexpected value type in getOrigin()") ? static_cast<void
> (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1684, __PRETTY_FUNCTION__))
1684 "Unexpected value type in getOrigin()")(((isa<Instruction>(V) || isa<Argument>(V)) &&
"Unexpected value type in getOrigin()") ? static_cast<void
> (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1684, __PRETTY_FUNCTION__))
;
1685 if (Instruction *I = dyn_cast<Instruction>(V)) {
1686 if (I->getMetadata("nosanitize"))
1687 return getCleanOrigin();
1688 }
1689 Value *Origin = OriginMap[V];
1690 assert(Origin && "Missing origin")((Origin && "Missing origin") ? static_cast<void>
(0) : __assert_fail ("Origin && \"Missing origin\"",
"/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1690, __PRETTY_FUNCTION__))
;
1691 return Origin;
1692 }
1693
1694 /// Get the origin for i-th argument of the instruction I.
1695 Value *getOrigin(Instruction *I, int i) {
1696 return getOrigin(I->getOperand(i));
1697 }
1698
1699 /// Remember the place where a shadow check should be inserted.
1700 ///
1701 /// This location will be later instrumented with a check that will print a
1702 /// UMR warning in runtime if the shadow value is not 0.
1703 void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1704 assert(Shadow)((Shadow) ? static_cast<void> (0) : __assert_fail ("Shadow"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1704, __PRETTY_FUNCTION__))
;
1705 if (!InsertChecks) return;
1706#ifndef NDEBUG
1707 Type *ShadowTy = Shadow->getType();
1708 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) &&(((isa<IntegerType>(ShadowTy) || isa<VectorType>(
ShadowTy)) && "Can only insert checks for integer and vector shadow types"
) ? static_cast<void> (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) && \"Can only insert checks for integer and vector shadow types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1709, __PRETTY_FUNCTION__))
1709 "Can only insert checks for integer and vector shadow types")(((isa<IntegerType>(ShadowTy) || isa<VectorType>(
ShadowTy)) && "Can only insert checks for integer and vector shadow types"
) ? static_cast<void> (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) && \"Can only insert checks for integer and vector shadow types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1709, __PRETTY_FUNCTION__))
;
1710#endif
1711 InstrumentationList.push_back(
1712 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1713 }
1714
1715 /// Remember the place where a shadow check should be inserted.
1716 ///
1717 /// This location will be later instrumented with a check that will print a
1718 /// UMR warning in runtime if the value is not fully defined.
1719 void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1720 assert(Val)((Val) ? static_cast<void> (0) : __assert_fail ("Val", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1720, __PRETTY_FUNCTION__))
;
1721 Value *Shadow, *Origin;
1722 if (ClCheckConstantShadow) {
1723 Shadow = getShadow(Val);
1724 if (!Shadow) return;
1725 Origin = getOrigin(Val);
1726 } else {
1727 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1728 if (!Shadow) return;
1729 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1730 }
1731 insertShadowCheck(Shadow, Origin, OrigIns);
1732 }
1733
1734 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
1735 switch (a) {
1736 case AtomicOrdering::NotAtomic:
1737 return AtomicOrdering::NotAtomic;
1738 case AtomicOrdering::Unordered:
1739 case AtomicOrdering::Monotonic:
1740 case AtomicOrdering::Release:
1741 return AtomicOrdering::Release;
1742 case AtomicOrdering::Acquire:
1743 case AtomicOrdering::AcquireRelease:
1744 return AtomicOrdering::AcquireRelease;
1745 case AtomicOrdering::SequentiallyConsistent:
1746 return AtomicOrdering::SequentiallyConsistent;
1747 }
1748 llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1748)
;
1749 }
1750
1751 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
1752 switch (a) {
1753 case AtomicOrdering::NotAtomic:
1754 return AtomicOrdering::NotAtomic;
1755 case AtomicOrdering::Unordered:
1756 case AtomicOrdering::Monotonic:
1757 case AtomicOrdering::Acquire:
1758 return AtomicOrdering::Acquire;
1759 case AtomicOrdering::Release:
1760 case AtomicOrdering::AcquireRelease:
1761 return AtomicOrdering::AcquireRelease;
1762 case AtomicOrdering::SequentiallyConsistent:
1763 return AtomicOrdering::SequentiallyConsistent;
1764 }
1765 llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1765)
;
1766 }
1767
1768 // ------------------- Visitors.
1769 using InstVisitor<MemorySanitizerVisitor>::visit;
1770 void visit(Instruction &I) {
1771 if (!I.getMetadata("nosanitize"))
1772 InstVisitor<MemorySanitizerVisitor>::visit(I);
1773 }
1774
1775 /// Instrument LoadInst
1776 ///
1777 /// Loads the corresponding shadow and (optionally) origin.
1778 /// Optionally, checks that the load address is fully defined.
1779 void visitLoadInst(LoadInst &I) {
1780 assert(I.getType()->isSized() && "Load type must have size")((I.getType()->isSized() && "Load type must have size"
) ? static_cast<void> (0) : __assert_fail ("I.getType()->isSized() && \"Load type must have size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1780, __PRETTY_FUNCTION__))
;
1781 assert(!I.getMetadata("nosanitize"))((!I.getMetadata("nosanitize")) ? static_cast<void> (0)
: __assert_fail ("!I.getMetadata(\"nosanitize\")", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1781, __PRETTY_FUNCTION__))
;
1782 IRBuilder<> IRB(I.getNextNode());
1783 Type *ShadowTy = getShadowTy(&I);
1784 Value *Addr = I.getPointerOperand();
1785 Value *ShadowPtr, *OriginPtr;
1786 unsigned Alignment = I.getAlignment();
1787 if (PropagateShadow) {
1788 std::tie(ShadowPtr, OriginPtr) =
1789 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
1790 setShadow(&I,
1791 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
1792 } else {
1793 setShadow(&I, getCleanShadow(&I));
1794 }
1795
1796 if (ClCheckAccessAddress)
1797 insertShadowCheck(I.getPointerOperand(), &I);
1798
1799 if (I.isAtomic())
1800 I.setOrdering(addAcquireOrdering(I.getOrdering()));
1801
1802 if (MS.TrackOrigins) {
1803 if (PropagateShadow) {
1804 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1805 setOrigin(
1806 &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
1807 } else {
1808 setOrigin(&I, getCleanOrigin());
1809 }
1810 }
1811 }
1812
1813 /// Instrument StoreInst
1814 ///
1815 /// Stores the corresponding shadow and (optionally) origin.
1816 /// Optionally, checks that the store address is fully defined.
1817 void visitStoreInst(StoreInst &I) {
1818 StoreList.push_back(&I);
1819 if (ClCheckAccessAddress)
1820 insertShadowCheck(I.getPointerOperand(), &I);
1821 }
1822
1823 void handleCASOrRMW(Instruction &I) {
1824 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I))((isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>
(I)) ? static_cast<void> (0) : __assert_fail ("isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1824, __PRETTY_FUNCTION__))
;
1825
1826 IRBuilder<> IRB(&I);
1827 Value *Addr = I.getOperand(0);
1828 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, I.getType(),
1829 /*Alignment*/ 1, /*isStore*/ true)
1830 .first;
1831
1832 if (ClCheckAccessAddress)
1833 insertShadowCheck(Addr, &I);
1834
1835 // Only test the conditional argument of cmpxchg instruction.
1836 // The other argument can potentially be uninitialized, but we can not
1837 // detect this situation reliably without possible false positives.
1838 if (isa<AtomicCmpXchgInst>(I))
1839 insertShadowCheck(I.getOperand(1), &I);
1840
1841 IRB.CreateStore(getCleanShadow(&I), ShadowPtr);
1842
1843 setShadow(&I, getCleanShadow(&I));
1844 setOrigin(&I, getCleanOrigin());
1845 }
1846
1847 void visitAtomicRMWInst(AtomicRMWInst &I) {
1848 handleCASOrRMW(I);
1849 I.setOrdering(addReleaseOrdering(I.getOrdering()));
1850 }
1851
1852 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1853 handleCASOrRMW(I);
1854 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1855 }
1856
1857 // Vector manipulation.
1858 void visitExtractElementInst(ExtractElementInst &I) {
1859 insertShadowCheck(I.getOperand(1), &I);
1860 IRBuilder<> IRB(&I);
1861 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
1862 "_msprop"));
1863 setOrigin(&I, getOrigin(&I, 0));
1864 }
1865
1866 void visitInsertElementInst(InsertElementInst &I) {
1867 insertShadowCheck(I.getOperand(2), &I);
1868 IRBuilder<> IRB(&I);
1869 setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
1870 I.getOperand(2), "_msprop"));
1871 setOriginForNaryOp(I);
1872 }
1873
1874 void visitShuffleVectorInst(ShuffleVectorInst &I) {
1875 insertShadowCheck(I.getOperand(2), &I);
1876 IRBuilder<> IRB(&I);
1877 setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
1878 I.getOperand(2), "_msprop"));
1879 setOriginForNaryOp(I);
1880 }
1881
1882 // Casts.
1883 void visitSExtInst(SExtInst &I) {
1884 IRBuilder<> IRB(&I);
1885 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
1886 setOrigin(&I, getOrigin(&I, 0));
1887 }
1888
1889 void visitZExtInst(ZExtInst &I) {
1890 IRBuilder<> IRB(&I);
1891 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
1892 setOrigin(&I, getOrigin(&I, 0));
1893 }
1894
1895 void visitTruncInst(TruncInst &I) {
1896 IRBuilder<> IRB(&I);
1897 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
1898 setOrigin(&I, getOrigin(&I, 0));
1899 }
1900
1901 void visitBitCastInst(BitCastInst &I) {
1902 // Special case: if this is the bitcast (there is exactly 1 allowed) between
1903 // a musttail call and a ret, don't instrument. New instructions are not
1904 // allowed after a musttail call.
1905 if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
1906 if (CI->isMustTailCall())
1907 return;
1908 IRBuilder<> IRB(&I);
1909 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
1910 setOrigin(&I, getOrigin(&I, 0));
1911 }
1912
1913 void visitPtrToIntInst(PtrToIntInst &I) {
1914 IRBuilder<> IRB(&I);
1915 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1916 "_msprop_ptrtoint"));
1917 setOrigin(&I, getOrigin(&I, 0));
1918 }
1919
1920 void visitIntToPtrInst(IntToPtrInst &I) {
1921 IRBuilder<> IRB(&I);
1922 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1923 "_msprop_inttoptr"));
1924 setOrigin(&I, getOrigin(&I, 0));
1925 }
1926
1927 void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
1928 void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
1929 void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
1930 void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
1931 void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
1932 void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
1933
1934 /// Propagate shadow for bitwise AND.
1935 ///
1936 /// This code is exact, i.e. if, for example, a bit in the left argument
1937 /// is defined and 0, then neither the value not definedness of the
1938 /// corresponding bit in B don't affect the resulting shadow.
1939 void visitAnd(BinaryOperator &I) {
1940 IRBuilder<> IRB(&I);
1941 // "And" of 0 and a poisoned value results in unpoisoned value.
1942 // 1&1 => 1; 0&1 => 0; p&1 => p;
1943 // 1&0 => 0; 0&0 => 0; p&0 => 0;
1944 // 1&p => p; 0&p => 0; p&p => p;
1945 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
1946 Value *S1 = getShadow(&I, 0);
1947 Value *S2 = getShadow(&I, 1);
1948 Value *V1 = I.getOperand(0);
1949 Value *V2 = I.getOperand(1);
1950 if (V1->getType() != S1->getType()) {
1951 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1952 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1953 }
1954 Value *S1S2 = IRB.CreateAnd(S1, S2);
1955 Value *V1S2 = IRB.CreateAnd(V1, S2);
1956 Value *S1V2 = IRB.CreateAnd(S1, V2);
1957 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
1958 setOriginForNaryOp(I);
1959 }
1960
1961 void visitOr(BinaryOperator &I) {
1962 IRBuilder<> IRB(&I);
1963 // "Or" of 1 and a poisoned value results in unpoisoned value.
1964 // 1|1 => 1; 0|1 => 1; p|1 => 1;
1965 // 1|0 => 1; 0|0 => 0; p|0 => p;
1966 // 1|p => 1; 0|p => p; p|p => p;
1967 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
1968 Value *S1 = getShadow(&I, 0);
1969 Value *S2 = getShadow(&I, 1);
1970 Value *V1 = IRB.CreateNot(I.getOperand(0));
1971 Value *V2 = IRB.CreateNot(I.getOperand(1));
1972 if (V1->getType() != S1->getType()) {
1973 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1974 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1975 }
1976 Value *S1S2 = IRB.CreateAnd(S1, S2);
1977 Value *V1S2 = IRB.CreateAnd(V1, S2);
1978 Value *S1V2 = IRB.CreateAnd(S1, V2);
1979 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
1980 setOriginForNaryOp(I);
1981 }
1982
1983 /// Default propagation of shadow and/or origin.
1984 ///
1985 /// This class implements the general case of shadow propagation, used in all
1986 /// cases where we don't know and/or don't care about what the operation
1987 /// actually does. It converts all input shadow values to a common type
1988 /// (extending or truncating as necessary), and bitwise OR's them.
1989 ///
1990 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
1991 /// fully initialized), and less prone to false positives.
1992 ///
1993 /// This class also implements the general case of origin propagation. For a
1994 /// Nary operation, result origin is set to the origin of an argument that is
1995 /// not entirely initialized. If there is more than one such arguments, the
1996 /// rightmost of them is picked. It does not matter which one is picked if all
1997 /// arguments are initialized.
1998 template <bool CombineShadow>
1999 class Combiner {
2000 Value *Shadow = nullptr;
2001 Value *Origin = nullptr;
2002 IRBuilder<> &IRB;
2003 MemorySanitizerVisitor *MSV;
2004
2005 public:
2006 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2007 : IRB(IRB), MSV(MSV) {}
2008
2009 /// Add a pair of shadow and origin values to the mix.
2010 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2011 if (CombineShadow) {
2012 assert(OpShadow)((OpShadow) ? static_cast<void> (0) : __assert_fail ("OpShadow"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2012, __PRETTY_FUNCTION__))
;
2013 if (!Shadow)
2014 Shadow = OpShadow;
2015 else {
2016 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2017 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
2018 }
2019 }
2020
2021 if (MSV->MS.TrackOrigins) {
2022 assert(OpOrigin)((OpOrigin) ? static_cast<void> (0) : __assert_fail ("OpOrigin"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2022, __PRETTY_FUNCTION__))
;
2023 if (!Origin) {
2024 Origin = OpOrigin;
2025 } else {
2026 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2027 // No point in adding something that might result in 0 origin value.
2028 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2029 Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
2030 Value *Cond =
2031 IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
2032 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2033 }
2034 }
2035 }
2036 return *this;
2037 }
2038
2039 /// Add an application value to the mix.
2040 Combiner &Add(Value *V) {
2041 Value *OpShadow = MSV->getShadow(V);
2042 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2043 return Add(OpShadow, OpOrigin);
2044 }
2045
2046 /// Set the current combined values as the given instruction's shadow
2047 /// and origin.
2048 void Done(Instruction *I) {
2049 if (CombineShadow) {
2050 assert(Shadow)((Shadow) ? static_cast<void> (0) : __assert_fail ("Shadow"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2050, __PRETTY_FUNCTION__))
;
2051 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
2052 MSV->setShadow(I, Shadow);
2053 }
2054 if (MSV->MS.TrackOrigins) {
2055 assert(Origin)((Origin) ? static_cast<void> (0) : __assert_fail ("Origin"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2055, __PRETTY_FUNCTION__))
;
2056 MSV->setOrigin(I, Origin);
2057 }
2058 }
2059 };
2060
2061 using ShadowAndOriginCombiner = Combiner<true>;
2062 using OriginCombiner = Combiner<false>;
2063
2064 /// Propagate origin for arbitrary operation.
2065 void setOriginForNaryOp(Instruction &I) {
2066 if (!MS.TrackOrigins) return;
2067 IRBuilder<> IRB(&I);
2068 OriginCombiner OC(this, IRB);
2069 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
2070 OC.Add(OI->get());
2071 OC.Done(&I);
2072 }
2073
2074 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2075 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&((!(Ty->isVectorTy() && Ty->getScalarType()->
isPointerTy()) && "Vector of pointers is not a valid shadow type"
) ? static_cast<void> (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2076, __PRETTY_FUNCTION__))
2076 "Vector of pointers is not a valid shadow type")((!(Ty->isVectorTy() && Ty->getScalarType()->
isPointerTy()) && "Vector of pointers is not a valid shadow type"
) ? static_cast<void> (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2076, __PRETTY_FUNCTION__))
;
2077 return Ty->isVectorTy() ?
2078 Ty->getVectorNumElements() * Ty->getScalarSizeInBits() :
2079 Ty->getPrimitiveSizeInBits();
2080 }
2081
2082 /// Cast between two shadow types, extending or truncating as
2083 /// necessary.
2084 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2085 bool Signed = false) {
2086 Type *srcTy = V->getType();
2087 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2088 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2089 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2090 return IRB.CreateICmpNE(V, getCleanShadow(V));
2091
2092 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2093 return IRB.CreateIntCast(V, dstTy, Signed);
2094 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2095 dstTy->getVectorNumElements() == srcTy->getVectorNumElements())
2096 return IRB.CreateIntCast(V, dstTy, Signed);
2097 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
2098 Value *V2 =
2099 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
2100 return IRB.CreateBitCast(V2, dstTy);
2101 // TODO: handle struct types.
2102 }
2103
2104 /// Cast an application value to the type of its own shadow.
2105 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2106 Type *ShadowTy = getShadowTy(V);
2107 if (V->getType() == ShadowTy)
2108 return V;
2109 if (V->getType()->isPtrOrPtrVectorTy())
2110 return IRB.CreatePtrToInt(V, ShadowTy);
2111 else
2112 return IRB.CreateBitCast(V, ShadowTy);
2113 }
2114
2115 /// Propagate shadow for arbitrary operation.
2116 void handleShadowOr(Instruction &I) {
2117 IRBuilder<> IRB(&I);
2118 ShadowAndOriginCombiner SC(this, IRB);
2119 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
2120 SC.Add(OI->get());
2121 SC.Done(&I);
2122 }
2123
2124 void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
2125
2126 // Handle multiplication by constant.
2127 //
2128 // Handle a special case of multiplication by constant that may have one or
2129 // more zeros in the lower bits. This makes corresponding number of lower bits
2130 // of the result zero as well. We model it by shifting the other operand
2131 // shadow left by the required number of bits. Effectively, we transform
2132 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2133 // We use multiplication by 2**N instead of shift to cover the case of
2134 // multiplication by 0, which may occur in some elements of a vector operand.
2135 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2136 Value *OtherArg) {
2137 Constant *ShadowMul;
2138 Type *Ty = ConstArg->getType();
2139 if (Ty->isVectorTy()) {
2140 unsigned NumElements = Ty->getVectorNumElements();
2141 Type *EltTy = Ty->getSequentialElementType();
2142 SmallVector<Constant *, 16> Elements;
2143 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2144 if (ConstantInt *Elt =
2145 dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
2146 const APInt &V = Elt->getValue();
2147 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2148 Elements.push_back(ConstantInt::get(EltTy, V2));
2149 } else {
2150 Elements.push_back(ConstantInt::get(EltTy, 1));
2151 }
2152 }
2153 ShadowMul = ConstantVector::get(Elements);
2154 } else {
2155 if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2156 const APInt &V = Elt->getValue();
2157 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2158 ShadowMul = ConstantInt::get(Ty, V2);
2159 } else {
2160 ShadowMul = ConstantInt::get(Ty, 1);
2161 }
2162 }
2163
2164 IRBuilder<> IRB(&I);
2165 setShadow(&I,
2166 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
2167 setOrigin(&I, getOrigin(OtherArg));
2168 }
2169
2170 void visitMul(BinaryOperator &I) {
2171 Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
2172 Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
2173 if (constOp0 && !constOp1)
2174 handleMulByConstant(I, constOp0, I.getOperand(1));
2175 else if (constOp1 && !constOp0)
2176 handleMulByConstant(I, constOp1, I.getOperand(0));
2177 else
2178 handleShadowOr(I);
2179 }
2180
2181 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
2182 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
2183 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
2184 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
2185 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
2186 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
2187
2188 void handleIntegerDiv(Instruction &I) {
2189 IRBuilder<> IRB(&I);
2190 // Strict on the second argument.
2191 insertShadowCheck(I.getOperand(1), &I);
2192 setShadow(&I, getShadow(&I, 0));
2193 setOrigin(&I, getOrigin(&I, 0));
2194 }
2195
2196 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2197 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2198 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
2199 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
2200
2201 // Floating point division is side-effect free. We can not require that the
2202 // divisor is fully initialized and must propagate shadow. See PR37523.
2203 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
2204 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
2205
2206 /// Instrument == and != comparisons.
2207 ///
2208 /// Sometimes the comparison result is known even if some of the bits of the
2209 /// arguments are not.
2210 void handleEqualityComparison(ICmpInst &I) {
2211 IRBuilder<> IRB(&I);
2212 Value *A = I.getOperand(0);
2213 Value *B = I.getOperand(1);
2214 Value *Sa = getShadow(A);
2215 Value *Sb = getShadow(B);
2216
2217 // Get rid of pointers and vectors of pointers.
2218 // For ints (and vectors of ints), types of A and Sa match,
2219 // and this is a no-op.
2220 A = IRB.CreatePointerCast(A, Sa->getType());
2221 B = IRB.CreatePointerCast(B, Sb->getType());
2222
2223 // A == B <==> (C = A^B) == 0
2224 // A != B <==> (C = A^B) != 0
2225 // Sc = Sa | Sb
2226 Value *C = IRB.CreateXor(A, B);
2227 Value *Sc = IRB.CreateOr(Sa, Sb);
2228 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2229 // Result is defined if one of the following is true
2230 // * there is a defined 1 bit in C
2231 // * C is fully defined
2232 // Si = !(C & ~Sc) && Sc
2233 Value *Zero = Constant::getNullValue(Sc->getType());
2234 Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
2235 Value *Si =
2236 IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
2237 IRB.CreateICmpEQ(
2238 IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
2239 Si->setName("_msprop_icmp");
2240 setShadow(&I, Si);
2241 setOriginForNaryOp(I);
2242 }
2243
2244 /// Build the lowest possible value of V, taking into account V's
2245 /// uninitialized bits.
2246 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2247 bool isSigned) {
2248 if (isSigned) {
2249 // Split shadow into sign bit and other bits.
2250 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2251 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2252 // Maximise the undefined shadow bit, minimize other undefined bits.
2253 return
2254 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
2255 } else {
2256 // Minimize undefined bits.
2257 return IRB.CreateAnd(A, IRB.CreateNot(Sa));
2258 }
2259 }
2260
2261 /// Build the highest possible value of V, taking into account V's
2262 /// uninitialized bits.
2263 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2264 bool isSigned) {
2265 if (isSigned) {
2266 // Split shadow into sign bit and other bits.
2267 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2268 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2269 // Minimise the undefined shadow bit, maximise other undefined bits.
2270 return
2271 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
2272 } else {
2273 // Maximize undefined bits.
2274 return IRB.CreateOr(A, Sa);
2275 }
2276 }
2277
2278 /// Instrument relational comparisons.
2279 ///
2280 /// This function does exact shadow propagation for all relational
2281 /// comparisons of integers, pointers and vectors of those.
2282 /// FIXME: output seems suboptimal when one of the operands is a constant
2283 void handleRelationalComparisonExact(ICmpInst &I) {
2284 IRBuilder<> IRB(&I);
2285 Value *A = I.getOperand(0);
2286 Value *B = I.getOperand(1);
2287 Value *Sa = getShadow(A);
2288 Value *Sb = getShadow(B);
2289
2290 // Get rid of pointers and vectors of pointers.
2291 // For ints (and vectors of ints), types of A and Sa match,
2292 // and this is a no-op.
2293 A = IRB.CreatePointerCast(A, Sa->getType());
2294 B = IRB.CreatePointerCast(B, Sb->getType());
2295
2296 // Let [a0, a1] be the interval of possible values of A, taking into account
2297 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
2298 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
2299 bool IsSigned = I.isSigned();
2300 Value *S1 = IRB.CreateICmp(I.getPredicate(),
2301 getLowestPossibleValue(IRB, A, Sa, IsSigned),
2302 getHighestPossibleValue(IRB, B, Sb, IsSigned));
2303 Value *S2 = IRB.CreateICmp(I.getPredicate(),
2304 getHighestPossibleValue(IRB, A, Sa, IsSigned),
2305 getLowestPossibleValue(IRB, B, Sb, IsSigned));
2306 Value *Si = IRB.CreateXor(S1, S2);
2307 setShadow(&I, Si);
2308 setOriginForNaryOp(I);
2309 }
2310
2311 /// Instrument signed relational comparisons.
2312 ///
2313 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
2314 /// bit of the shadow. Everything else is delegated to handleShadowOr().
2315 void handleSignedRelationalComparison(ICmpInst &I) {
2316 Constant *constOp;
2317 Value *op = nullptr;
2318 CmpInst::Predicate pre;
2319 if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
2320 op = I.getOperand(0);
2321 pre = I.getPredicate();
2322 } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
2323 op = I.getOperand(1);
2324 pre = I.getSwappedPredicate();
2325 } else {
2326 handleShadowOr(I);
2327 return;
2328 }
2329
2330 if ((constOp->isNullValue() &&
2331 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2332 (constOp->isAllOnesValue() &&
2333 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2334 IRBuilder<> IRB(&I);
2335 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2336 "_msprop_icmp_s");
2337 setShadow(&I, Shadow);
2338 setOrigin(&I, getOrigin(op));
2339 } else {
2340 handleShadowOr(I);
2341 }
2342 }
2343
2344 void visitICmpInst(ICmpInst &I) {
2345 if (!ClHandleICmp) {
2346 handleShadowOr(I);
2347 return;
2348 }
2349 if (I.isEquality()) {
2350 handleEqualityComparison(I);
2351 return;
2352 }
2353
2354 assert(I.isRelational())((I.isRelational()) ? static_cast<void> (0) : __assert_fail
("I.isRelational()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2354, __PRETTY_FUNCTION__))
;
2355 if (ClHandleICmpExact) {
2356 handleRelationalComparisonExact(I);
2357 return;
2358 }
2359 if (I.isSigned()) {
2360 handleSignedRelationalComparison(I);
2361 return;
2362 }
2363
2364 assert(I.isUnsigned())((I.isUnsigned()) ? static_cast<void> (0) : __assert_fail
("I.isUnsigned()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2364, __PRETTY_FUNCTION__))
;
2365 if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
2366 handleRelationalComparisonExact(I);
2367 return;
2368 }
2369
2370 handleShadowOr(I);
2371 }
2372
2373 void visitFCmpInst(FCmpInst &I) {
2374 handleShadowOr(I);
2375 }
2376
2377 void handleShift(BinaryOperator &I) {
2378 IRBuilder<> IRB(&I);
2379 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2380 // Otherwise perform the same shift on S1.
2381 Value *S1 = getShadow(&I, 0);
2382 Value *S2 = getShadow(&I, 1);
2383 Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
2384 S2->getType());
2385 Value *V2 = I.getOperand(1);
2386 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2387 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2388 setOriginForNaryOp(I);
2389 }
2390
2391 void visitShl(BinaryOperator &I) { handleShift(I); }
2392 void visitAShr(BinaryOperator &I) { handleShift(I); }
2393 void visitLShr(BinaryOperator &I) { handleShift(I); }
2394
2395 /// Instrument llvm.memmove
2396 ///
2397 /// At this point we don't know if llvm.memmove will be inlined or not.
2398 /// If we don't instrument it and it gets inlined,
2399 /// our interceptor will not kick in and we will lose the memmove.
2400 /// If we instrument the call here, but it does not get inlined,
2401 /// we will memove the shadow twice: which is bad in case
2402 /// of overlapping regions. So, we simply lower the intrinsic to a call.
2403 ///
2404 /// Similar situation exists for memcpy and memset.
2405 void visitMemMoveInst(MemMoveInst &I) {
2406 IRBuilder<> IRB(&I);
2407 IRB.CreateCall(
2408 MS.MemmoveFn,
2409 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2410 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2411 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2412 I.eraseFromParent();
2413 }
2414
2415 // Similar to memmove: avoid copying shadow twice.
2416 // This is somewhat unfortunate as it may slowdown small constant memcpys.
2417 // FIXME: consider doing manual inline for small constant sizes and proper
2418 // alignment.
2419 void visitMemCpyInst(MemCpyInst &I) {
2420 IRBuilder<> IRB(&I);
2421 IRB.CreateCall(
2422 MS.MemcpyFn,
2423 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2424 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2425 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2426 I.eraseFromParent();
2427 }
2428
2429 // Same as memcpy.
2430 void visitMemSetInst(MemSetInst &I) {
2431 IRBuilder<> IRB(&I);
2432 IRB.CreateCall(
2433 MS.MemsetFn,
2434 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2435 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2436 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2437 I.eraseFromParent();
2438 }
2439
2440 void visitVAStartInst(VAStartInst &I) {
2441 VAHelper->visitVAStartInst(I);
2442 }
2443
2444 void visitVACopyInst(VACopyInst &I) {
2445 VAHelper->visitVACopyInst(I);
2446 }
2447
2448 /// Handle vector store-like intrinsics.
2449 ///
2450 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
2451 /// has 1 pointer argument and 1 vector argument, returns void.
2452 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2453 IRBuilder<> IRB(&I);
2454 Value* Addr = I.getArgOperand(0);
2455 Value *Shadow = getShadow(&I, 1);
2456 Value *ShadowPtr, *OriginPtr;
2457
2458 // We don't know the pointer alignment (could be unaligned SSE store!).
2459 // Have to assume to worst case.
2460 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2461 Addr, IRB, Shadow->getType(), /*Alignment*/ 1, /*isStore*/ true);
2462 IRB.CreateAlignedStore(Shadow, ShadowPtr, 1);
2463
2464 if (ClCheckAccessAddress)
2465 insertShadowCheck(Addr, &I);
2466
2467 // FIXME: factor out common code from materializeStores
2468 if (MS.TrackOrigins) IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2469 return true;
2470 }
2471
2472 /// Handle vector load-like intrinsics.
2473 ///
2474 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
2475 /// has 1 pointer argument, returns a vector.
2476 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2477 IRBuilder<> IRB(&I);
2478 Value *Addr = I.getArgOperand(0);
2479
2480 Type *ShadowTy = getShadowTy(&I);
2481 Value *ShadowPtr, *OriginPtr;
2482 if (PropagateShadow) {
2483 // We don't know the pointer alignment (could be unaligned SSE load!).
2484 // Have to assume to worst case.
2485 unsigned Alignment = 1;
2486 std::tie(ShadowPtr, OriginPtr) =
2487 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2488 setShadow(&I,
2489 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2490 } else {
2491 setShadow(&I, getCleanShadow(&I));
2492 }
2493
2494 if (ClCheckAccessAddress)
2495 insertShadowCheck(Addr, &I);
2496
2497 if (MS.TrackOrigins) {
2498 if (PropagateShadow)
2499 setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
2500 else
2501 setOrigin(&I, getCleanOrigin());
2502 }
2503 return true;
2504 }
2505
2506 /// Handle (SIMD arithmetic)-like intrinsics.
2507 ///
2508 /// Instrument intrinsics with any number of arguments of the same type,
2509 /// equal to the return type. The type should be simple (no aggregates or
2510 /// pointers; vectors are fine).
2511 /// Caller guarantees that this intrinsic does not access memory.
2512 bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2513 Type *RetTy = I.getType();
2514 if (!(RetTy->isIntOrIntVectorTy() ||
2515 RetTy->isFPOrFPVectorTy() ||
2516 RetTy->isX86_MMXTy()))
2517 return false;
2518
2519 unsigned NumArgOperands = I.getNumArgOperands();
2520
2521 for (unsigned i = 0; i < NumArgOperands; ++i) {
2522 Type *Ty = I.getArgOperand(i)->getType();
2523 if (Ty != RetTy)
2524 return false;
2525 }
2526
2527 IRBuilder<> IRB(&I);
2528 ShadowAndOriginCombiner SC(this, IRB);
2529 for (unsigned i = 0; i < NumArgOperands; ++i)
2530 SC.Add(I.getArgOperand(i));
2531 SC.Done(&I);
2532
2533 return true;
2534 }
2535
2536 /// Heuristically instrument unknown intrinsics.
2537 ///
2538 /// The main purpose of this code is to do something reasonable with all
2539 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2540 /// We recognize several classes of intrinsics by their argument types and
2541 /// ModRefBehaviour and apply special intrumentation when we are reasonably
2542 /// sure that we know what the intrinsic does.
2543 ///
2544 /// We special-case intrinsics where this approach fails. See llvm.bswap
2545 /// handling as an example of that.
2546 bool handleUnknownIntrinsic(IntrinsicInst &I) {
2547 unsigned NumArgOperands = I.getNumArgOperands();
2548 if (NumArgOperands == 0)
2549 return false;
2550
2551 if (NumArgOperands == 2 &&
2552 I.getArgOperand(0)->getType()->isPointerTy() &&
2553 I.getArgOperand(1)->getType()->isVectorTy() &&
2554 I.getType()->isVoidTy() &&
2555 !I.onlyReadsMemory()) {
2556 // This looks like a vector store.
2557 return handleVectorStoreIntrinsic(I);
2558 }
2559
2560 if (NumArgOperands == 1 &&
2561 I.getArgOperand(0)->getType()->isPointerTy() &&
2562 I.getType()->isVectorTy() &&
2563 I.onlyReadsMemory()) {
2564 // This looks like a vector load.
2565 return handleVectorLoadIntrinsic(I);
2566 }
2567
2568 if (I.doesNotAccessMemory())
2569 if (maybeHandleSimpleNomemIntrinsic(I))
2570 return true;
2571
2572 // FIXME: detect and handle SSE maskstore/maskload
2573 return false;
2574 }
2575
2576 void handleInvariantGroup(IntrinsicInst &I) {
2577 setShadow(&I, getShadow(&I, 0));
2578 setOrigin(&I, getOrigin(&I, 0));
2579 }
2580
2581 void handleLifetimeStart(IntrinsicInst &I) {
2582 if (!PoisonStack)
2583 return;
2584 DenseMap<Value *, AllocaInst *> AllocaForValue;
2585 AllocaInst *AI =
2586 llvm::findAllocaForValue(I.getArgOperand(1), AllocaForValue);
2587 if (!AI)
2588 InstrumentLifetimeStart = false;
2589 LifetimeStartList.push_back(std::make_pair(&I, AI));
2590 }
2591
2592 void handleBswap(IntrinsicInst &I) {
2593 IRBuilder<> IRB(&I);
2594 Value *Op = I.getArgOperand(0);
2595 Type *OpType = Op->getType();
2596 Function *BswapFunc = Intrinsic::getDeclaration(
2597 F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2598 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2599 setOrigin(&I, getOrigin(Op));
2600 }
2601
2602 // Instrument vector convert instrinsic.
2603 //
2604 // This function instruments intrinsics like cvtsi2ss:
2605 // %Out = int_xxx_cvtyyy(%ConvertOp)
2606 // or
2607 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2608 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2609 // number \p Out elements, and (if has 2 arguments) copies the rest of the
2610 // elements from \p CopyOp.
2611 // In most cases conversion involves floating-point value which may trigger a
2612 // hardware exception when not fully initialized. For this reason we require
2613 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2614 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2615 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2616 // return a fully initialized value.
2617 void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) {
2618 IRBuilder<> IRB(&I);
2619 Value *CopyOp, *ConvertOp;
2620
2621 switch (I.getNumArgOperands()) {
2622 case 3:
2623 assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode")((isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantInt>(I.getArgOperand(2)) && \"Invalid rounding mode\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2623, __PRETTY_FUNCTION__))
;
2624 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2625 case 2:
2626 CopyOp = I.getArgOperand(0);
2627 ConvertOp = I.getArgOperand(1);
2628 break;
2629 case 1:
2630 ConvertOp = I.getArgOperand(0);
2631 CopyOp = nullptr;
2632 break;
2633 default:
2634 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.")::llvm::llvm_unreachable_internal("Cvt intrinsic with unsupported number of arguments."
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2634)
;
2635 }
2636
2637 // The first *NumUsedElements* elements of ConvertOp are converted to the
2638 // same number of output elements. The rest of the output is copied from
2639 // CopyOp, or (if not available) filled with zeroes.
2640 // Combine shadow for elements of ConvertOp that are used in this operation,
2641 // and insert a check.
2642 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2643 // int->any conversion.
2644 Value *ConvertShadow = getShadow(ConvertOp);
2645 Value *AggShadow = nullptr;
2646 if (ConvertOp->getType()->isVectorTy()) {
2647 AggShadow = IRB.CreateExtractElement(
2648 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2649 for (int i = 1; i < NumUsedElements; ++i) {
2650 Value *MoreShadow = IRB.CreateExtractElement(
2651 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2652 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2653 }
2654 } else {
2655 AggShadow = ConvertShadow;
2656 }
2657 assert(AggShadow->getType()->isIntegerTy())((AggShadow->getType()->isIntegerTy()) ? static_cast<
void> (0) : __assert_fail ("AggShadow->getType()->isIntegerTy()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2657, __PRETTY_FUNCTION__))
;
2658 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2659
2660 // Build result shadow by zero-filling parts of CopyOp shadow that come from
2661 // ConvertOp.
2662 if (CopyOp) {
2663 assert(CopyOp->getType() == I.getType())((CopyOp->getType() == I.getType()) ? static_cast<void>
(0) : __assert_fail ("CopyOp->getType() == I.getType()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2663, __PRETTY_FUNCTION__))
;
2664 assert(CopyOp->getType()->isVectorTy())((CopyOp->getType()->isVectorTy()) ? static_cast<void
> (0) : __assert_fail ("CopyOp->getType()->isVectorTy()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2664, __PRETTY_FUNCTION__))
;
2665 Value *ResultShadow = getShadow(CopyOp);
2666 Type *EltTy = ResultShadow->getType()->getVectorElementType();
2667 for (int i = 0; i < NumUsedElements; ++i) {
2668 ResultShadow = IRB.CreateInsertElement(
2669 ResultShadow, ConstantInt::getNullValue(EltTy),
2670 ConstantInt::get(IRB.getInt32Ty(), i));
2671 }
2672 setShadow(&I, ResultShadow);
2673 setOrigin(&I, getOrigin(CopyOp));
2674 } else {
2675 setShadow(&I, getCleanShadow(&I));
2676 setOrigin(&I, getCleanOrigin());
2677 }
2678 }
2679
2680 // Given a scalar or vector, extract lower 64 bits (or less), and return all
2681 // zeroes if it is zero, and all ones otherwise.
2682 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2683 if (S->getType()->isVectorTy())
2684 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2685 assert(S->getType()->getPrimitiveSizeInBits() <= 64)((S->getType()->getPrimitiveSizeInBits() <= 64) ? static_cast
<void> (0) : __assert_fail ("S->getType()->getPrimitiveSizeInBits() <= 64"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2685, __PRETTY_FUNCTION__))
;
2686 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2687 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2688 }
2689
2690 // Given a vector, extract its first element, and return all
2691 // zeroes if it is zero, and all ones otherwise.
2692 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2693 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2694 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2695 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2696 }
2697
2698 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2699 Type *T = S->getType();
2700 assert(T->isVectorTy())((T->isVectorTy()) ? static_cast<void> (0) : __assert_fail
("T->isVectorTy()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2700, __PRETTY_FUNCTION__))
;
2701 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2702 return IRB.CreateSExt(S2, T);
2703 }
2704
2705 // Instrument vector shift instrinsic.
2706 //
2707 // This function instruments intrinsics like int_x86_avx2_psll_w.
2708 // Intrinsic shifts %In by %ShiftSize bits.
2709 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2710 // size, and the rest is ignored. Behavior is defined even if shift size is
2711 // greater than register (or field) width.
2712 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2713 assert(I.getNumArgOperands() == 2)((I.getNumArgOperands() == 2) ? static_cast<void> (0) :
__assert_fail ("I.getNumArgOperands() == 2", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2713, __PRETTY_FUNCTION__))
;
2714 IRBuilder<> IRB(&I);
2715 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2716 // Otherwise perform the same shift on S1.
2717 Value *S1 = getShadow(&I, 0);
2718 Value *S2 = getShadow(&I, 1);
2719 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2720 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2721 Value *V1 = I.getOperand(0);
2722 Value *V2 = I.getOperand(1);
2723 Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledValue(),
2724 {IRB.CreateBitCast(S1, V1->getType()), V2});
2725 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2726 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2727 setOriginForNaryOp(I);
2728 }
2729
2730 // Get an X86_MMX-sized vector type.
2731 Type *getMMXVectorTy(unsigned EltSizeInBits) {
2732 const unsigned X86_MMXSizeInBits = 64;
2733 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&((EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits
) == 0 && "Illegal MMX vector element size") ? static_cast
<void> (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2734, __PRETTY_FUNCTION__))
2734 "Illegal MMX vector element size")((EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits
) == 0 && "Illegal MMX vector element size") ? static_cast
<void> (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2734, __PRETTY_FUNCTION__))
;
2735 return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2736 X86_MMXSizeInBits / EltSizeInBits);
2737 }
2738
2739 // Returns a signed counterpart for an (un)signed-saturate-and-pack
2740 // intrinsic.
2741 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2742 switch (id) {
2743 case Intrinsic::x86_sse2_packsswb_128:
2744 case Intrinsic::x86_sse2_packuswb_128:
2745 return Intrinsic::x86_sse2_packsswb_128;
2746
2747 case Intrinsic::x86_sse2_packssdw_128:
2748 case Intrinsic::x86_sse41_packusdw:
2749 return Intrinsic::x86_sse2_packssdw_128;
2750
2751 case Intrinsic::x86_avx2_packsswb:
2752 case Intrinsic::x86_avx2_packuswb:
2753 return Intrinsic::x86_avx2_packsswb;
2754
2755 case Intrinsic::x86_avx2_packssdw:
2756 case Intrinsic::x86_avx2_packusdw:
2757 return Intrinsic::x86_avx2_packssdw;
2758
2759 case Intrinsic::x86_mmx_packsswb:
2760 case Intrinsic::x86_mmx_packuswb:
2761 return Intrinsic::x86_mmx_packsswb;
2762
2763 case Intrinsic::x86_mmx_packssdw:
2764 return Intrinsic::x86_mmx_packssdw;
2765 default:
2766 llvm_unreachable("unexpected intrinsic id")::llvm::llvm_unreachable_internal("unexpected intrinsic id", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2766)
;
2767 }
2768 }
2769
2770 // Instrument vector pack instrinsic.
2771 //
2772 // This function instruments intrinsics like x86_mmx_packsswb, that
2773 // packs elements of 2 input vectors into half as many bits with saturation.
2774 // Shadow is propagated with the signed variant of the same intrinsic applied
2775 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2776 // EltSizeInBits is used only for x86mmx arguments.
2777 void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2778 assert(I.getNumArgOperands() == 2)((I.getNumArgOperands() == 2) ? static_cast<void> (0) :
__assert_fail ("I.getNumArgOperands() == 2", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2778, __PRETTY_FUNCTION__))
;
2779 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2780 IRBuilder<> IRB(&I);
2781 Value *S1 = getShadow(&I, 0);
2782 Value *S2 = getShadow(&I, 1);
2783 assert(isX86_MMX || S1->getType()->isVectorTy())((isX86_MMX || S1->getType()->isVectorTy()) ? static_cast
<void> (0) : __assert_fail ("isX86_MMX || S1->getType()->isVectorTy()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2783, __PRETTY_FUNCTION__))
;
2784
2785 // SExt and ICmpNE below must apply to individual elements of input vectors.
2786 // In case of x86mmx arguments, cast them to appropriate vector types and
2787 // back.
2788 Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
2789 if (isX86_MMX) {
2790 S1 = IRB.CreateBitCast(S1, T);
2791 S2 = IRB.CreateBitCast(S2, T);
2792 }
2793 Value *S1_ext = IRB.CreateSExt(
2794 IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
2795 Value *S2_ext = IRB.CreateSExt(
2796 IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
2797 if (isX86_MMX) {
2798 Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2799 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2800 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2801 }
2802
2803 Function *ShadowFn = Intrinsic::getDeclaration(
2804 F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2805
2806 Value *S =
2807 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2808 if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
2809 setShadow(&I, S);
2810 setOriginForNaryOp(I);
2811 }
2812
2813 // Instrument sum-of-absolute-differencies intrinsic.
2814 void handleVectorSadIntrinsic(IntrinsicInst &I) {
2815 const unsigned SignificantBitsPerResultElement = 16;
2816 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2817 Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
2818 unsigned ZeroBitsPerResultElement =
2819 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2820
2821 IRBuilder<> IRB(&I);
2822 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2823 S = IRB.CreateBitCast(S, ResTy);
2824 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2825 ResTy);
2826 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2827 S = IRB.CreateBitCast(S, getShadowTy(&I));
2828 setShadow(&I, S);
2829 setOriginForNaryOp(I);
2830 }
2831
2832 // Instrument multiply-add intrinsic.
2833 void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2834 unsigned EltSizeInBits = 0) {
2835 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2836 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
2837 IRBuilder<> IRB(&I);
2838 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2839 S = IRB.CreateBitCast(S, ResTy);
2840 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2841 ResTy);
2842 S = IRB.CreateBitCast(S, getShadowTy(&I));
2843 setShadow(&I, S);
2844 setOriginForNaryOp(I);
2845 }
2846
2847 // Instrument compare-packed intrinsic.
2848 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
2849 // all-ones shadow.
2850 void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
2851 IRBuilder<> IRB(&I);
2852 Type *ResTy = getShadowTy(&I);
2853 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2854 Value *S = IRB.CreateSExt(
2855 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
2856 setShadow(&I, S);
2857 setOriginForNaryOp(I);
2858 }
2859
2860 // Instrument compare-scalar intrinsic.
2861 // This handles both cmp* intrinsics which return the result in the first
2862 // element of a vector, and comi* which return the result as i32.
2863 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
2864 IRBuilder<> IRB(&I);
2865 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2866 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
2867 setShadow(&I, S);
2868 setOriginForNaryOp(I);
2869 }
2870
2871 void handleStmxcsr(IntrinsicInst &I) {
2872 IRBuilder<> IRB(&I);
2873 Value* Addr = I.getArgOperand(0);
2874 Type *Ty = IRB.getInt32Ty();
2875 Value *ShadowPtr =
2876 getShadowOriginPtr(Addr, IRB, Ty, /*Alignment*/ 1, /*isStore*/ true)
2877 .first;
2878
2879 IRB.CreateStore(getCleanShadow(Ty),
2880 IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
2881
2882 if (ClCheckAccessAddress)
2883 insertShadowCheck(Addr, &I);
2884 }
2885
2886 void handleLdmxcsr(IntrinsicInst &I) {
2887 if (!InsertChecks) return;
2888
2889 IRBuilder<> IRB(&I);
2890 Value *Addr = I.getArgOperand(0);
2891 Type *Ty = IRB.getInt32Ty();
2892 unsigned Alignment = 1;
2893 Value *ShadowPtr, *OriginPtr;
2894 std::tie(ShadowPtr, OriginPtr) =
2895 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
2896
2897 if (ClCheckAccessAddress)
2898 insertShadowCheck(Addr, &I);
2899
2900 Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
2901 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
2902 : getCleanOrigin();
2903 insertShadowCheck(Shadow, Origin, &I);
2904 }
2905
2906 void handleMaskedStore(IntrinsicInst &I) {
2907 IRBuilder<> IRB(&I);
2908 Value *V = I.getArgOperand(0);
2909 Value *Addr = I.getArgOperand(1);
2910 unsigned Align = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
2911 Value *Mask = I.getArgOperand(3);
2912 Value *Shadow = getShadow(V);
2913
2914 Value *ShadowPtr;
2915 Value *OriginPtr;
2916 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2917 Addr, IRB, Shadow->getType(), Align, /*isStore*/ true);
2918
2919 if (ClCheckAccessAddress) {
2920 insertShadowCheck(Addr, &I);
2921 // Uninitialized mask is kind of like uninitialized address, but not as
2922 // scary.
2923 insertShadowCheck(Mask, &I);
2924 }
2925
2926 IRB.CreateMaskedStore(Shadow, ShadowPtr, Align, Mask);
2927
2928 if (MS.TrackOrigins) {
2929 auto &DL = F.getParent()->getDataLayout();
2930 paintOrigin(IRB, getOrigin(V), OriginPtr,
2931 DL.getTypeStoreSize(Shadow->getType()),
2932 std::max(Align, kMinOriginAlignment));
2933 }
2934 }
2935
2936 bool handleMaskedLoad(IntrinsicInst &I) {
2937 IRBuilder<> IRB(&I);
2938 Value *Addr = I.getArgOperand(0);
2939 unsigned Align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
2940 Value *Mask = I.getArgOperand(2);
2941 Value *PassThru = I.getArgOperand(3);
2942
2943 Type *ShadowTy = getShadowTy(&I);
2944 Value *ShadowPtr, *OriginPtr;
2945 if (PropagateShadow) {
2946 std::tie(ShadowPtr, OriginPtr) =
2947 getShadowOriginPtr(Addr, IRB, ShadowTy, Align, /*isStore*/ false);
2948 setShadow(&I, IRB.CreateMaskedLoad(ShadowPtr, Align, Mask,
2949 getShadow(PassThru), "_msmaskedld"));
2950 } else {
2951 setShadow(&I, getCleanShadow(&I));
2952 }
2953
2954 if (ClCheckAccessAddress) {
2955 insertShadowCheck(Addr, &I);
2956 insertShadowCheck(Mask, &I);
2957 }
2958
2959 if (MS.TrackOrigins) {
2960 if (PropagateShadow) {
2961 // Choose between PassThru's and the loaded value's origins.
2962 Value *MaskedPassThruShadow = IRB.CreateAnd(
2963 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
2964
2965 Value *Acc = IRB.CreateExtractElement(
2966 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2967 for (int i = 1, N = PassThru->getType()->getVectorNumElements(); i < N;
2968 ++i) {
2969 Value *More = IRB.CreateExtractElement(
2970 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2971 Acc = IRB.CreateOr(Acc, More);
2972 }
2973
2974 Value *Origin = IRB.CreateSelect(
2975 IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
2976 getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr));
2977
2978 setOrigin(&I, Origin);
2979 } else {
2980 setOrigin(&I, getCleanOrigin());
2981 }
2982 }
2983 return true;
2984 }
2985
2986 // Instrument BMI / BMI2 intrinsics.
2987 // All of these intrinsics are Z = I(X, Y)
2988 // where the types of all operands and the result match, and are either i32 or i64.
2989 // The following instrumentation happens to work for all of them:
2990 // Sz = I(Sx, Y) | (sext (Sy != 0))
2991 void handleBmiIntrinsic(IntrinsicInst &I) {
2992 IRBuilder<> IRB(&I);
2993 Type *ShadowTy = getShadowTy(&I);
2994
2995 // If any bit of the mask operand is poisoned, then the whole thing is.
2996 Value *SMask = getShadow(&I, 1);
2997 SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
2998 ShadowTy);
2999 // Apply the same intrinsic to the shadow of the first operand.
3000 Value *S = IRB.CreateCall(I.getCalledFunction(),
3001 {getShadow(&I, 0), I.getOperand(1)});
3002 S = IRB.CreateOr(SMask, S);
3003 setShadow(&I, S);
3004 setOriginForNaryOp(I);
3005 }
3006
3007 void visitIntrinsicInst(IntrinsicInst &I) {
3008 switch (I.getIntrinsicID()) {
3009 case Intrinsic::lifetime_start:
3010 handleLifetimeStart(I);
3011 break;
3012 case Intrinsic::launder_invariant_group:
3013 case Intrinsic::strip_invariant_group:
3014 handleInvariantGroup(I);
3015 break;
3016 case Intrinsic::bswap:
3017 handleBswap(I);
3018 break;
3019 case Intrinsic::masked_store:
3020 handleMaskedStore(I);
3021 break;
3022 case Intrinsic::masked_load:
3023 handleMaskedLoad(I);
3024 break;
3025 case Intrinsic::x86_sse_stmxcsr:
3026 handleStmxcsr(I);
3027 break;
3028 case Intrinsic::x86_sse_ldmxcsr:
3029 handleLdmxcsr(I);
3030 break;
3031 case Intrinsic::x86_avx512_vcvtsd2usi64:
3032 case Intrinsic::x86_avx512_vcvtsd2usi32:
3033 case Intrinsic::x86_avx512_vcvtss2usi64:
3034 case Intrinsic::x86_avx512_vcvtss2usi32:
3035 case Intrinsic::x86_avx512_cvttss2usi64:
3036 case Intrinsic::x86_avx512_cvttss2usi:
3037 case Intrinsic::x86_avx512_cvttsd2usi64:
3038 case Intrinsic::x86_avx512_cvttsd2usi:
3039 case Intrinsic::x86_avx512_cvtusi2ss:
3040 case Intrinsic::x86_avx512_cvtusi642sd:
3041 case Intrinsic::x86_avx512_cvtusi642ss:
3042 case Intrinsic::x86_sse2_cvtsd2si64:
3043 case Intrinsic::x86_sse2_cvtsd2si:
3044 case Intrinsic::x86_sse2_cvtsd2ss:
3045 case Intrinsic::x86_sse2_cvttsd2si64:
3046 case Intrinsic::x86_sse2_cvttsd2si:
3047 case Intrinsic::x86_sse_cvtss2si64:
3048 case Intrinsic::x86_sse_cvtss2si:
3049 case Intrinsic::x86_sse_cvttss2si64:
3050 case Intrinsic::x86_sse_cvttss2si:
3051 handleVectorConvertIntrinsic(I, 1);
3052 break;
3053 case Intrinsic::x86_sse_cvtps2pi:
3054 case Intrinsic::x86_sse_cvttps2pi:
3055 handleVectorConvertIntrinsic(I, 2);
3056 break;
3057
3058 case Intrinsic::x86_avx512_psll_w_512:
3059 case Intrinsic::x86_avx512_psll_d_512:
3060 case Intrinsic::x86_avx512_psll_q_512:
3061 case Intrinsic::x86_avx512_pslli_w_512:
3062 case Intrinsic::x86_avx512_pslli_d_512:
3063 case Intrinsic::x86_avx512_pslli_q_512:
3064 case Intrinsic::x86_avx512_psrl_w_512:
3065 case Intrinsic::x86_avx512_psrl_d_512:
3066 case Intrinsic::x86_avx512_psrl_q_512:
3067 case Intrinsic::x86_avx512_psra_w_512:
3068 case Intrinsic::x86_avx512_psra_d_512:
3069 case Intrinsic::x86_avx512_psra_q_512:
3070 case Intrinsic::x86_avx512_psrli_w_512:
3071 case Intrinsic::x86_avx512_psrli_d_512:
3072 case Intrinsic::x86_avx512_psrli_q_512:
3073 case Intrinsic::x86_avx512_psrai_w_512:
3074 case Intrinsic::x86_avx512_psrai_d_512:
3075 case Intrinsic::x86_avx512_psrai_q_512:
3076 case Intrinsic::x86_avx512_psra_q_256:
3077 case Intrinsic::x86_avx512_psra_q_128:
3078 case Intrinsic::x86_avx512_psrai_q_256:
3079 case Intrinsic::x86_avx512_psrai_q_128:
3080 case Intrinsic::x86_avx2_psll_w:
3081 case Intrinsic::x86_avx2_psll_d:
3082 case Intrinsic::x86_avx2_psll_q:
3083 case Intrinsic::x86_avx2_pslli_w:
3084 case Intrinsic::x86_avx2_pslli_d:
3085 case Intrinsic::x86_avx2_pslli_q:
3086 case Intrinsic::x86_avx2_psrl_w:
3087 case Intrinsic::x86_avx2_psrl_d:
3088 case Intrinsic::x86_avx2_psrl_q:
3089 case Intrinsic::x86_avx2_psra_w:
3090 case Intrinsic::x86_avx2_psra_d:
3091 case Intrinsic::x86_avx2_psrli_w:
3092 case Intrinsic::x86_avx2_psrli_d:
3093 case Intrinsic::x86_avx2_psrli_q:
3094 case Intrinsic::x86_avx2_psrai_w:
3095 case Intrinsic::x86_avx2_psrai_d:
3096 case Intrinsic::x86_sse2_psll_w:
3097 case Intrinsic::x86_sse2_psll_d:
3098 case Intrinsic::x86_sse2_psll_q:
3099 case Intrinsic::x86_sse2_pslli_w:
3100 case Intrinsic::x86_sse2_pslli_d:
3101 case Intrinsic::x86_sse2_pslli_q:
3102 case Intrinsic::x86_sse2_psrl_w:
3103 case Intrinsic::x86_sse2_psrl_d:
3104 case Intrinsic::x86_sse2_psrl_q:
3105 case Intrinsic::x86_sse2_psra_w:
3106 case Intrinsic::x86_sse2_psra_d:
3107 case Intrinsic::x86_sse2_psrli_w:
3108 case Intrinsic::x86_sse2_psrli_d:
3109 case Intrinsic::x86_sse2_psrli_q:
3110 case Intrinsic::x86_sse2_psrai_w:
3111 case Intrinsic::x86_sse2_psrai_d:
3112 case Intrinsic::x86_mmx_psll_w:
3113 case Intrinsic::x86_mmx_psll_d:
3114 case Intrinsic::x86_mmx_psll_q:
3115 case Intrinsic::x86_mmx_pslli_w:
3116 case Intrinsic::x86_mmx_pslli_d:
3117 case Intrinsic::x86_mmx_pslli_q:
3118 case Intrinsic::x86_mmx_psrl_w:
3119 case Intrinsic::x86_mmx_psrl_d:
3120 case Intrinsic::x86_mmx_psrl_q:
3121 case Intrinsic::x86_mmx_psra_w:
3122 case Intrinsic::x86_mmx_psra_d:
3123 case Intrinsic::x86_mmx_psrli_w:
3124 case Intrinsic::x86_mmx_psrli_d:
3125 case Intrinsic::x86_mmx_psrli_q:
3126 case Intrinsic::x86_mmx_psrai_w:
3127 case Intrinsic::x86_mmx_psrai_d:
3128 handleVectorShiftIntrinsic(I, /* Variable */ false);
3129 break;
3130 case Intrinsic::x86_avx2_psllv_d:
3131 case Intrinsic::x86_avx2_psllv_d_256:
3132 case Intrinsic::x86_avx512_psllv_d_512:
3133 case Intrinsic::x86_avx2_psllv_q:
3134 case Intrinsic::x86_avx2_psllv_q_256:
3135 case Intrinsic::x86_avx512_psllv_q_512:
3136 case Intrinsic::x86_avx2_psrlv_d:
3137 case Intrinsic::x86_avx2_psrlv_d_256:
3138 case Intrinsic::x86_avx512_psrlv_d_512:
3139 case Intrinsic::x86_avx2_psrlv_q:
3140 case Intrinsic::x86_avx2_psrlv_q_256:
3141 case Intrinsic::x86_avx512_psrlv_q_512:
3142 case Intrinsic::x86_avx2_psrav_d:
3143 case Intrinsic::x86_avx2_psrav_d_256:
3144 case Intrinsic::x86_avx512_psrav_d_512:
3145 case Intrinsic::x86_avx512_psrav_q_128:
3146 case Intrinsic::x86_avx512_psrav_q_256:
3147 case Intrinsic::x86_avx512_psrav_q_512:
3148 handleVectorShiftIntrinsic(I, /* Variable */ true);
3149 break;
3150
3151 case Intrinsic::x86_sse2_packsswb_128:
3152 case Intrinsic::x86_sse2_packssdw_128:
3153 case Intrinsic::x86_sse2_packuswb_128:
3154 case Intrinsic::x86_sse41_packusdw:
3155 case Intrinsic::x86_avx2_packsswb:
3156 case Intrinsic::x86_avx2_packssdw:
3157 case Intrinsic::x86_avx2_packuswb:
3158 case Intrinsic::x86_avx2_packusdw:
3159 handleVectorPackIntrinsic(I);
3160 break;
3161
3162 case Intrinsic::x86_mmx_packsswb:
3163 case Intrinsic::x86_mmx_packuswb:
3164 handleVectorPackIntrinsic(I, 16);
3165 break;
3166
3167 case Intrinsic::x86_mmx_packssdw:
3168 handleVectorPackIntrinsic(I, 32);
3169 break;
3170
3171 case Intrinsic::x86_mmx_psad_bw:
3172 case Intrinsic::x86_sse2_psad_bw:
3173 case Intrinsic::x86_avx2_psad_bw:
3174 handleVectorSadIntrinsic(I);
3175 break;
3176
3177 case Intrinsic::x86_sse2_pmadd_wd:
3178 case Intrinsic::x86_avx2_pmadd_wd:
3179 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3180 case Intrinsic::x86_avx2_pmadd_ub_sw:
3181 handleVectorPmaddIntrinsic(I);
3182 break;
3183
3184 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3185 handleVectorPmaddIntrinsic(I, 8);
3186 break;
3187
3188 case Intrinsic::x86_mmx_pmadd_wd:
3189 handleVectorPmaddIntrinsic(I, 16);
3190 break;
3191
3192 case Intrinsic::x86_sse_cmp_ss:
3193 case Intrinsic::x86_sse2_cmp_sd:
3194 case Intrinsic::x86_sse_comieq_ss:
3195 case Intrinsic::x86_sse_comilt_ss:
3196 case Intrinsic::x86_sse_comile_ss:
3197 case Intrinsic::x86_sse_comigt_ss:
3198 case Intrinsic::x86_sse_comige_ss:
3199 case Intrinsic::x86_sse_comineq_ss:
3200 case Intrinsic::x86_sse_ucomieq_ss:
3201 case Intrinsic::x86_sse_ucomilt_ss:
3202 case Intrinsic::x86_sse_ucomile_ss:
3203 case Intrinsic::x86_sse_ucomigt_ss:
3204 case Intrinsic::x86_sse_ucomige_ss:
3205 case Intrinsic::x86_sse_ucomineq_ss:
3206 case Intrinsic::x86_sse2_comieq_sd:
3207 case Intrinsic::x86_sse2_comilt_sd:
3208 case Intrinsic::x86_sse2_comile_sd:
3209 case Intrinsic::x86_sse2_comigt_sd:
3210 case Intrinsic::x86_sse2_comige_sd:
3211 case Intrinsic::x86_sse2_comineq_sd:
3212 case Intrinsic::x86_sse2_ucomieq_sd:
3213 case Intrinsic::x86_sse2_ucomilt_sd:
3214 case Intrinsic::x86_sse2_ucomile_sd:
3215 case Intrinsic::x86_sse2_ucomigt_sd:
3216 case Intrinsic::x86_sse2_ucomige_sd:
3217 case Intrinsic::x86_sse2_ucomineq_sd:
3218 handleVectorCompareScalarIntrinsic(I);
3219 break;
3220
3221 case Intrinsic::x86_sse_cmp_ps:
3222 case Intrinsic::x86_sse2_cmp_pd:
3223 // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
3224 // generates reasonably looking IR that fails in the backend with "Do not
3225 // know how to split the result of this operator!".
3226 handleVectorComparePackedIntrinsic(I);
3227 break;
3228
3229 case Intrinsic::x86_bmi_bextr_32:
3230 case Intrinsic::x86_bmi_bextr_64:
3231 case Intrinsic::x86_bmi_bzhi_32:
3232 case Intrinsic::x86_bmi_bzhi_64:
3233 case Intrinsic::x86_bmi_pdep_32:
3234 case Intrinsic::x86_bmi_pdep_64:
3235 case Intrinsic::x86_bmi_pext_32:
3236 case Intrinsic::x86_bmi_pext_64:
3237 handleBmiIntrinsic(I);
3238 break;
3239
3240 case Intrinsic::is_constant:
3241 // The result of llvm.is.constant() is always defined.
3242 setShadow(&I, getCleanShadow(&I));
3243 setOrigin(&I, getCleanOrigin());
3244 break;
3245
3246 default:
3247 if (!handleUnknownIntrinsic(I))
3248 visitInstruction(I);
3249 break;
3250 }
3251 }
3252
3253 void visitCallSite(CallSite CS) {
3254 Instruction &I = *CS.getInstruction();
3255 assert(!I.getMetadata("nosanitize"))((!I.getMetadata("nosanitize")) ? static_cast<void> (0)
: __assert_fail ("!I.getMetadata(\"nosanitize\")", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3255, __PRETTY_FUNCTION__))
;
3256 assert((CS.isCall() || CS.isInvoke() || CS.isCallBr()) &&(((CS.isCall() || CS.isInvoke() || CS.isCallBr()) && "Unknown type of CallSite"
) ? static_cast<void> (0) : __assert_fail ("(CS.isCall() || CS.isInvoke() || CS.isCallBr()) && \"Unknown type of CallSite\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3257, __PRETTY_FUNCTION__))
3257 "Unknown type of CallSite")(((CS.isCall() || CS.isInvoke() || CS.isCallBr()) && "Unknown type of CallSite"
) ? static_cast<void> (0) : __assert_fail ("(CS.isCall() || CS.isInvoke() || CS.isCallBr()) && \"Unknown type of CallSite\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3257, __PRETTY_FUNCTION__))
;
3258 if (CS.isCallBr() || (CS.isCall() && cast<CallInst>(&I)->isInlineAsm())) {
3259 // For inline asm (either a call to asm function, or callbr instruction),
3260 // do the usual thing: check argument shadow and mark all outputs as
3261 // clean. Note that any side effects of the inline asm that are not
3262 // immediately visible in its constraints are not handled.
3263 if (ClHandleAsmConservative && MS.CompileKernel)
3264 visitAsmInstruction(I);
3265 else
3266 visitInstruction(I);
3267 return;
3268 }
3269 if (CS.isCall()) {
3270 CallInst *Call = cast<CallInst>(&I);
3271 assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere")((!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("!isa<IntrinsicInst>(&I) && \"intrinsics are handled elsewhere\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3271, __PRETTY_FUNCTION__))
;
3272
3273 // We are going to insert code that relies on the fact that the callee
3274 // will become a non-readonly function after it is instrumented by us. To
3275 // prevent this code from being optimized out, mark that function
3276 // non-readonly in advance.
3277 if (Function *Func = Call->getCalledFunction()) {
3278 // Clear out readonly/readnone attributes.
3279 AttrBuilder B;
3280 B.addAttribute(Attribute::ReadOnly)
3281 .addAttribute(Attribute::ReadNone)
3282 .addAttribute(Attribute::WriteOnly)
3283 .addAttribute(Attribute::ArgMemOnly)
3284 .addAttribute(Attribute::Speculatable);
3285 Func->removeAttributes(AttributeList::FunctionIndex, B);
3286 }
3287
3288 maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
3289 }
3290 IRBuilder<> IRB(&I);
3291
3292 unsigned ArgOffset = 0;
3293 LLVM_DEBUG(dbgs() << " CallSite: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " CallSite: " << I <<
"\n"; } } while (false)
;
3294 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3295 ArgIt != End; ++ArgIt) {
3296 Value *A = *ArgIt;
3297 unsigned i = ArgIt - CS.arg_begin();
3298 if (!A->getType()->isSized()) {
3299 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Arg " << i << " is not sized: "
<< I << "\n"; } } while (false)
;
3300 continue;
3301 }
3302 unsigned Size = 0;
3303 Value *Store = nullptr;
3304 // Compute the Shadow for arg even if it is ByVal, because
3305 // in that case getShadow() will copy the actual arg shadow to
3306 // __msan_param_tls.
3307 Value *ArgShadow = getShadow(A);
3308 Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
3309 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *Ado { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Arg#" << i << ": "
<< *A << " Shadow: " << *ArgShadow <<
"\n"; } } while (false)
3310 << " Shadow: " << *ArgShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Arg#" << i << ": "
<< *A << " Shadow: " << *ArgShadow <<
"\n"; } } while (false)
;
3311 bool ArgIsInitialized = false;
3312 const DataLayout &DL = F.getParent()->getDataLayout();
3313 if (CS.paramHasAttr(i, Attribute::ByVal)) {
3314 assert(A->getType()->isPointerTy() &&((A->getType()->isPointerTy() && "ByVal argument is not a pointer!"
) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3315, __PRETTY_FUNCTION__))
3315 "ByVal argument is not a pointer!")((A->getType()->isPointerTy() && "ByVal argument is not a pointer!"
) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3315, __PRETTY_FUNCTION__))
;
3316 Size = DL.getTypeAllocSize(A->getType()->getPointerElementType());
3317 if (ArgOffset + Size > kParamTLSSize) break;
3318 unsigned ParamAlignment = CS.getParamAlignment(i);
3319 unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment);
3320 Value *AShadowPtr =
3321 getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
3322 /*isStore*/ false)
3323 .first;
3324
3325 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
3326 Alignment, Size);
3327 // TODO(glider): need to copy origins.
3328 } else {
3329 Size = DL.getTypeAllocSize(A->getType());
3330 if (ArgOffset + Size > kParamTLSSize) break;
3331 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
3332 kShadowTLSAlignment);
3333 Constant *Cst = dyn_cast<Constant>(ArgShadow);
3334 if (Cst && Cst->isNullValue()) ArgIsInitialized = true;
3335 }
3336 if (MS.TrackOrigins && !ArgIsInitialized)
3337 IRB.CreateStore(getOrigin(A),
3338 getOriginPtrForArgument(A, IRB, ArgOffset));
3339 (void)Store;
3340 assert(Size != 0 && Store != nullptr)((Size != 0 && Store != nullptr) ? static_cast<void
> (0) : __assert_fail ("Size != 0 && Store != nullptr"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3340, __PRETTY_FUNCTION__))
;
3341 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Param:" << *Store <<
"\n"; } } while (false)
;
3342 ArgOffset += alignTo(Size, 8);
3343 }
3344 LLVM_DEBUG(dbgs() << " done with call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " done with call args\n"; } } while
(false)
;
3345
3346 FunctionType *FT = CS.getFunctionType();
3347 if (FT->isVarArg()) {
3348 VAHelper->visitCallSite(CS, IRB);
3349 }
3350
3351 // Now, get the shadow for the RetVal.
3352 if (!I.getType()->isSized()) return;
3353 // Don't emit the epilogue for musttail call returns.
3354 if (CS.isCall() && cast<CallInst>(&I)->isMustTailCall()) return;
3355 IRBuilder<> IRBBefore(&I);
3356 // Until we have full dynamic coverage, make sure the retval shadow is 0.
3357 Value *Base = getShadowPtrForRetval(&I, IRBBefore);
3358 IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, kShadowTLSAlignment);
3359 BasicBlock::iterator NextInsn;
3360 if (CS.isCall()) {
3361 NextInsn = ++I.getIterator();
3362 assert(NextInsn != I.getParent()->end())((NextInsn != I.getParent()->end()) ? static_cast<void>
(0) : __assert_fail ("NextInsn != I.getParent()->end()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3362, __PRETTY_FUNCTION__))
;
3363 } else {
3364 BasicBlock *NormalDest = cast<InvokeInst>(&I)->getNormalDest();
3365 if (!NormalDest->getSinglePredecessor()) {
3366 // FIXME: this case is tricky, so we are just conservative here.
3367 // Perhaps we need to split the edge between this BB and NormalDest,
3368 // but a naive attempt to use SplitEdge leads to a crash.
3369 setShadow(&I, getCleanShadow(&I));
3370 setOrigin(&I, getCleanOrigin());
3371 return;
3372 }
3373 // FIXME: NextInsn is likely in a basic block that has not been visited yet.
3374 // Anything inserted there will be instrumented by MSan later!
3375 NextInsn = NormalDest->getFirstInsertionPt();
3376 assert(NextInsn != NormalDest->end() &&((NextInsn != NormalDest->end() && "Could not find insertion point for retval shadow load"
) ? static_cast<void> (0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3377, __PRETTY_FUNCTION__))
3377 "Could not find insertion point for retval shadow load")((NextInsn != NormalDest->end() && "Could not find insertion point for retval shadow load"
) ? static_cast<void> (0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3377, __PRETTY_FUNCTION__))
;
3378 }
3379 IRBuilder<> IRBAfter(&*NextInsn);
3380 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
3381 getShadowTy(&I), getShadowPtrForRetval(&I, IRBAfter),
3382 kShadowTLSAlignment, "_msret");
3383 setShadow(&I, RetvalShadow);
3384 if (MS.TrackOrigins)
3385 setOrigin(&I, IRBAfter.CreateLoad(MS.OriginTy,
3386 getOriginPtrForRetval(IRBAfter)));
3387 }
3388
3389 bool isAMustTailRetVal(Value *RetVal) {
3390 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
3391 RetVal = I->getOperand(0);
3392 }
3393 if (auto *I = dyn_cast<CallInst>(RetVal)) {
3394 return I->isMustTailCall();
3395 }
3396 return false;
3397 }
3398
3399 void visitReturnInst(ReturnInst &I) {
3400 IRBuilder<> IRB(&I);
3401 Value *RetVal = I.getReturnValue();
3402 if (!RetVal) return;
3403 // Don't emit the epilogue for musttail call returns.
3404 if (isAMustTailRetVal(RetVal)) return;
3405 Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
3406 if (CheckReturnValue) {
3407 insertShadowCheck(RetVal, &I);
3408 Value *Shadow = getCleanShadow(RetVal);
3409 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3410 } else {
3411 Value *Shadow = getShadow(RetVal);
3412 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3413 if (MS.TrackOrigins)
3414 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
3415 }
3416 }
3417
3418 void visitPHINode(PHINode &I) {
3419 IRBuilder<> IRB(&I);
3420 if (!PropagateShadow) {
3421 setShadow(&I, getCleanShadow(&I));
3422 setOrigin(&I, getCleanOrigin());
3423 return;
3424 }
3425
3426 ShadowPHINodes.push_back(&I);
3427 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
3428 "_msphi_s"));
3429 if (MS.TrackOrigins)
3430 setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
3431 "_msphi_o"));
3432 }
3433
3434 Value *getLocalVarDescription(AllocaInst &I) {
3435 SmallString<2048> StackDescriptionStorage;
3436 raw_svector_ostream StackDescription(StackDescriptionStorage);
3437 // We create a string with a description of the stack allocation and
3438 // pass it into __msan_set_alloca_origin.
3439 // It will be printed by the run-time if stack-originated UMR is found.
3440 // The first 4 bytes of the string are set to '----' and will be replaced
3441 // by __msan_va_arg_overflow_size_tls at the first call.
3442 StackDescription << "----" << I.getName() << "@" << F.getName();
3443 return createPrivateNonConstGlobalForString(*F.getParent(),
3444 StackDescription.str());
3445 }
3446
3447 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3448 if (PoisonStack && ClPoisonStackWithCall) {
3449 IRB.CreateCall(MS.MsanPoisonStackFn,
3450 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3451 } else {
3452 Value *ShadowBase, *OriginBase;
3453 std::tie(ShadowBase, OriginBase) =
3454 getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(), 1, /*isStore*/ true);
3455
3456 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
3457 IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlignment());
3458 }
3459
3460 if (PoisonStack && MS.TrackOrigins) {
3461 Value *Descr = getLocalVarDescription(I);
3462 IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
3463 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3464 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
3465 IRB.CreatePointerCast(&F, MS.IntptrTy)});
3466 }
3467 }
3468
3469 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3470 Value *Descr = getLocalVarDescription(I);
3471 if (PoisonStack) {
3472 IRB.CreateCall(MS.MsanPoisonAllocaFn,
3473 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3474 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
3475 } else {
3476 IRB.CreateCall(MS.MsanUnpoisonAllocaFn,
3477 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3478 }
3479 }
3480
3481 void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
3482 if (!InsPoint)
3483 InsPoint = &I;
3484 IRBuilder<> IRB(InsPoint->getNextNode());
3485 const DataLayout &DL = F.getParent()->getDataLayout();
3486 uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
3487 Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
3488 if (I.isArrayAllocation())
3489 Len = IRB.CreateMul(Len, I.getArraySize());
3490
3491 if (MS.CompileKernel)
3492 poisonAllocaKmsan(I, IRB, Len);
3493 else
3494 poisonAllocaUserspace(I, IRB, Len);
3495 }
3496
3497 void visitAllocaInst(AllocaInst &I) {
3498 setShadow(&I, getCleanShadow(&I));
3499 setOrigin(&I, getCleanOrigin());
3500 // We'll get to this alloca later unless it's poisoned at the corresponding
3501 // llvm.lifetime.start.
3502 AllocaSet.insert(&I);
3503 }
3504
3505 void visitSelectInst(SelectInst& I) {
3506 IRBuilder<> IRB(&I);
3507 // a = select b, c, d
3508 Value *B = I.getCondition();
3509 Value *C = I.getTrueValue();
3510 Value *D = I.getFalseValue();
3511 Value *Sb = getShadow(B);
3512 Value *Sc = getShadow(C);
3513 Value *Sd = getShadow(D);
3514
3515 // Result shadow if condition shadow is 0.
3516 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
3517 Value *Sa1;
3518 if (I.getType()->isAggregateType()) {
3519 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
3520 // an extra "select". This results in much more compact IR.
3521 // Sa = select Sb, poisoned, (select b, Sc, Sd)
3522 Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
3523 } else {
3524 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
3525 // If Sb (condition is poisoned), look for bits in c and d that are equal
3526 // and both unpoisoned.
3527 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
3528
3529 // Cast arguments to shadow-compatible type.
3530 C = CreateAppToShadowCast(IRB, C);
3531 D = CreateAppToShadowCast(IRB, D);
3532
3533 // Result shadow if condition shadow is 1.
3534 Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd});
3535 }
3536 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
3537 setShadow(&I, Sa);
3538 if (MS.TrackOrigins) {
3539 // Origins are always i32, so any vector conditions must be flattened.
3540 // FIXME: consider tracking vector origins for app vectors?
3541 if (B->getType()->isVectorTy()) {
3542 Type *FlatTy = getShadowTyNoVec(B->getType());
3543 B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
3544 ConstantInt::getNullValue(FlatTy));
3545 Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
3546 ConstantInt::getNullValue(FlatTy));
3547 }
3548 // a = select b, c, d
3549 // Oa = Sb ? Ob : (b ? Oc : Od)
3550 setOrigin(
3551 &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
3552 IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
3553 getOrigin(I.getFalseValue()))));
3554 }
3555 }
3556
3557 void visitLandingPadInst(LandingPadInst &I) {
3558 // Do nothing.
3559 // See https://github.com/google/sanitizers/issues/504
3560 setShadow(&I, getCleanShadow(&I));
3561 setOrigin(&I, getCleanOrigin());
3562 }
3563
3564 void visitCatchSwitchInst(CatchSwitchInst &I) {
3565 setShadow(&I, getCleanShadow(&I));
3566 setOrigin(&I, getCleanOrigin());
3567 }
3568
3569 void visitFuncletPadInst(FuncletPadInst &I) {
3570 setShadow(&I, getCleanShadow(&I));
3571 setOrigin(&I, getCleanOrigin());
3572 }
3573
3574 void visitGetElementPtrInst(GetElementPtrInst &I) {
3575 handleShadowOr(I);
3576 }
3577
3578 void visitExtractValueInst(ExtractValueInst &I) {
3579 IRBuilder<> IRB(&I);
3580 Value *Agg = I.getAggregateOperand();
3581 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "ExtractValue: " << I <<
"\n"; } } while (false)
;
3582 Value *AggShadow = getShadow(Agg);
3583 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " AggShadow: " << *AggShadow
<< "\n"; } } while (false)
;
3584 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
3585 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ResShadow: " << *ResShadow
<< "\n"; } } while (false)
;
3586 setShadow(&I, ResShadow);
3587 setOriginForNaryOp(I);
3588 }
3589
3590 void visitInsertValueInst(InsertValueInst &I) {
3591 IRBuilder<> IRB(&I);
3592 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "InsertValue: " << I <<
"\n"; } } while (false)
;
3593 Value *AggShadow = getShadow(I.getAggregateOperand());
3594 Value *InsShadow = getShadow(I.getInsertedValueOperand());
3595 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " AggShadow: " << *AggShadow
<< "\n"; } } while (false)
;
3596 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " InsShadow: " << *InsShadow
<< "\n"; } } while (false)
;
3597 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
3598 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Res: " << *Res <<
"\n"; } } while (false)
;
3599 setShadow(&I, Res);
3600 setOriginForNaryOp(I);
3601 }
3602
3603 void dumpInst(Instruction &I) {
3604 if (CallInst *CI = dyn_cast<CallInst>(&I)) {
3605 errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
3606 } else {
3607 errs() << "ZZZ " << I.getOpcodeName() << "\n";
3608 }
3609 errs() << "QQQ " << I << "\n";
3610 }
3611
3612 void visitResumeInst(ResumeInst &I) {
3613 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Resume: " << I << "\n"
; } } while (false)
;
3614 // Nothing to do here.
3615 }
3616
3617 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
3618 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "CleanupReturn: " << CRI <<
"\n"; } } while (false)
;
3619 // Nothing to do here.
3620 }
3621
3622 void visitCatchReturnInst(CatchReturnInst &CRI) {
3623 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "CatchReturn: " << CRI <<
"\n"; } } while (false)
;
3624 // Nothing to do here.
3625 }
3626
3627 void instrumentAsmArgument(Value *Operand, Instruction &I, IRBuilder<> &IRB,
3628 const DataLayout &DL, bool isOutput) {
3629 // For each assembly argument, we check its value for being initialized.
3630 // If the argument is a pointer, we assume it points to a single element
3631 // of the corresponding type (or to a 8-byte word, if the type is unsized).
3632 // Each such pointer is instrumented with a call to the runtime library.
3633 Type *OpType = Operand->getType();
3634 // Check the operand value itself.
3635 insertShadowCheck(Operand, &I);
3636 if (!OpType->isPointerTy() || !isOutput) {
3637 assert(!isOutput)((!isOutput) ? static_cast<void> (0) : __assert_fail ("!isOutput"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3637, __PRETTY_FUNCTION__))
;
3638 return;
3639 }
3640 Type *ElType = OpType->getPointerElementType();
3641 if (!ElType->isSized())
3642 return;
3643 int Size = DL.getTypeStoreSize(ElType);
3644 Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy());
3645 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
3646 IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
3647 }
3648
3649 /// Get the number of output arguments returned by pointers.
3650 int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
3651 int NumRetOutputs = 0;
3652 int NumOutputs = 0;
3653 Type *RetTy = cast<Value>(CB)->getType();
3654 if (!RetTy->isVoidTy()) {
3655 // Register outputs are returned via the CallInst return value.
3656 auto *ST = dyn_cast<StructType>(RetTy);
3657 if (ST)
3658 NumRetOutputs = ST->getNumElements();
3659 else
3660 NumRetOutputs = 1;
3661 }
3662 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
3663 for (size_t i = 0, n = Constraints.size(); i < n; i++) {
3664 InlineAsm::ConstraintInfo Info = Constraints[i];
3665 switch (Info.Type) {
3666 case InlineAsm::isOutput:
3667 NumOutputs++;
3668 break;
3669 default:
3670 break;
3671 }
3672 }
3673 return NumOutputs - NumRetOutputs;
3674 }
3675
3676 void visitAsmInstruction(Instruction &I) {
3677 // Conservative inline assembly handling: check for poisoned shadow of
3678 // asm() arguments, then unpoison the result and all the memory locations
3679 // pointed to by those arguments.
3680 // An inline asm() statement in C++ contains lists of input and output
3681 // arguments used by the assembly code. These are mapped to operands of the
3682 // CallInst as follows:
3683 // - nR register outputs ("=r) are returned by value in a single structure
3684 // (SSA value of the CallInst);
3685 // - nO other outputs ("=m" and others) are returned by pointer as first
3686 // nO operands of the CallInst;
3687 // - nI inputs ("r", "m" and others) are passed to CallInst as the
3688 // remaining nI operands.
3689 // The total number of asm() arguments in the source is nR+nO+nI, and the
3690 // corresponding CallInst has nO+nI+1 operands (the last operand is the
3691 // function to be called).
3692 const DataLayout &DL = F.getParent()->getDataLayout();
3693 CallBase *CB = cast<CallBase>(&I);
3694 IRBuilder<> IRB(&I);
3695 InlineAsm *IA = cast<InlineAsm>(CB->getCalledValue());
3696 int OutputArgs = getNumOutputArgs(IA, CB);
3697 // The last operand of a CallInst is the function itself.
3698 int NumOperands = CB->getNumOperands() - 1;
3699
3700 // Check input arguments. Doing so before unpoisoning output arguments, so
3701 // that we won't overwrite uninit values before checking them.
3702 for (int i = OutputArgs; i < NumOperands; i++) {
3703 Value *Operand = CB->getOperand(i);
3704 instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ false);
3705 }
3706 // Unpoison output arguments. This must happen before the actual InlineAsm
3707 // call, so that the shadow for memory published in the asm() statement
3708 // remains valid.
3709 for (int i = 0; i < OutputArgs; i++) {
3710 Value *Operand = CB->getOperand(i);
3711 instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ true);
3712 }
3713
3714 setShadow(&I, getCleanShadow(&I));
3715 setOrigin(&I, getCleanOrigin());
3716 }
3717
3718 void visitInstruction(Instruction &I) {
3719 // Everything else: stop propagating and check for poisoned shadow.
3720 if (ClDumpStrictInstructions)
3721 dumpInst(I);
3722 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "DEFAULT: " << I << "\n"
; } } while (false)
;
3723 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
3724 Value *Operand = I.getOperand(i);
3725 if (Operand->getType()->isSized())
3726 insertShadowCheck(Operand, &I);
3727 }
3728 setShadow(&I, getCleanShadow(&I));
3729 setOrigin(&I, getCleanOrigin());
3730 }
3731};
3732
3733/// AMD64-specific implementation of VarArgHelper.
3734struct VarArgAMD64Helper : public VarArgHelper {
3735 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
3736 // See a comment in visitCallSite for more details.
3737 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
3738 static const unsigned AMD64FpEndOffsetSSE = 176;
3739 // If SSE is disabled, fp_offset in va_list is zero.
3740 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
3741
3742 unsigned AMD64FpEndOffset;
3743 Function &F;
3744 MemorySanitizer &MS;
3745 MemorySanitizerVisitor &MSV;
3746 Value *VAArgTLSCopy = nullptr;
3747 Value *VAArgTLSOriginCopy = nullptr;
3748 Value *VAArgOverflowSize = nullptr;
3749
3750 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3751
3752 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
3753
3754 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
3755 MemorySanitizerVisitor &MSV)
3756 : F(F), MS(MS), MSV(MSV) {
3757 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
3758 for (const auto &Attr : F.getAttributes().getFnAttributes()) {
3759 if (Attr.isStringAttribute() &&
3760 (Attr.getKindAsString() == "target-features")) {
3761 if (Attr.getValueAsString().contains("-sse"))
3762 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
3763 break;
3764 }
3765 }
3766 }
3767
3768 ArgKind classifyArgument(Value* arg) {
3769 // A very rough approximation of X86_64 argument classification rules.
3770 Type *T = arg->getType();
3771 if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
3772 return AK_FloatingPoint;
3773 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
3774 return AK_GeneralPurpose;
3775 if (T->isPointerTy())
3776 return AK_GeneralPurpose;
3777 return AK_Memory;
3778 }
3779
3780 // For VarArg functions, store the argument shadow in an ABI-specific format
3781 // that corresponds to va_list layout.
3782 // We do this because Clang lowers va_arg in the frontend, and this pass
3783 // only sees the low level code that deals with va_list internals.
3784 // A much easier alternative (provided that Clang emits va_arg instructions)
3785 // would have been to associate each live instance of va_list with a copy of
3786 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
3787 // order.
3788 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3789 unsigned GpOffset = 0;
3790 unsigned FpOffset = AMD64GpEndOffset;
3791 unsigned OverflowOffset = AMD64FpEndOffset;
3792 const DataLayout &DL = F.getParent()->getDataLayout();
3793 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3794 ArgIt != End; ++ArgIt) {
3795 Value *A = *ArgIt;
3796 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3797 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3798 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
3799 if (IsByVal) {
3800 // ByVal arguments always go to the overflow area.
3801 // Fixed arguments passed through the overflow area will be stepped
3802 // over by va_start, so don't count them towards the offset.
3803 if (IsFixed)
3804 continue;
3805 assert(A->getType()->isPointerTy())((A->getType()->isPointerTy()) ? static_cast<void>
(0) : __assert_fail ("A->getType()->isPointerTy()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3805, __PRETTY_FUNCTION__))
;
3806 Type *RealTy = A->getType()->getPointerElementType();
3807 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
3808 Value *ShadowBase = getShadowPtrForVAArgument(
3809 RealTy, IRB, OverflowOffset, alignTo(ArgSize, 8));
3810 Value *OriginBase = nullptr;
3811 if (MS.TrackOrigins)
3812 OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset);
3813 OverflowOffset += alignTo(ArgSize, 8);
3814 if (!ShadowBase)
3815 continue;
3816 Value *ShadowPtr, *OriginPtr;
3817 std::tie(ShadowPtr, OriginPtr) =
3818 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
3819 /*isStore*/ false);
3820
3821 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
3822 kShadowTLSAlignment, ArgSize);
3823 if (MS.TrackOrigins)
3824 IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
3825 kShadowTLSAlignment, ArgSize);
3826 } else {
3827 ArgKind AK = classifyArgument(A);
3828 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
3829 AK = AK_Memory;
3830 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
3831 AK = AK_Memory;
3832 Value *ShadowBase, *OriginBase = nullptr;
3833 switch (AK) {
3834 case AK_GeneralPurpose:
3835 ShadowBase =
3836 getShadowPtrForVAArgument(A->getType(), IRB, GpOffset, 8);
3837 if (MS.TrackOrigins)
3838 OriginBase =
3839 getOriginPtrForVAArgument(A->getType(), IRB, GpOffset);
3840 GpOffset += 8;
3841 break;
3842 case AK_FloatingPoint:
3843 ShadowBase =
3844 getShadowPtrForVAArgument(A->getType(), IRB, FpOffset, 16);
3845 if (MS.TrackOrigins)
3846 OriginBase =
3847 getOriginPtrForVAArgument(A->getType(), IRB, FpOffset);
3848 FpOffset += 16;
3849 break;
3850 case AK_Memory:
3851 if (IsFixed)
3852 continue;
3853 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3854 ShadowBase =
3855 getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, 8);
3856 if (MS.TrackOrigins)
3857 OriginBase =
3858 getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3859 OverflowOffset += alignTo(ArgSize, 8);
3860 }
3861 // Take fixed arguments into account for GpOffset and FpOffset,
3862 // but don't actually store shadows for them.
3863 // TODO(glider): don't call get*PtrForVAArgument() for them.
3864 if (IsFixed)
3865 continue;
3866 if (!ShadowBase)
3867 continue;
3868 Value *Shadow = MSV.getShadow(A);
3869 IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment);
3870 if (MS.TrackOrigins) {
3871 Value *Origin = MSV.getOrigin(A);
3872 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
3873 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
3874 std::max(kShadowTLSAlignment, kMinOriginAlignment));
3875 }
3876 }
3877 }
3878 Constant *OverflowSize =
3879 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
3880 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3881 }
3882
3883 /// Compute the shadow address for a given va_arg.
3884 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3885 unsigned ArgOffset, unsigned ArgSize) {
3886 // Make sure we don't overflow __msan_va_arg_tls.
3887 if (ArgOffset + ArgSize > kParamTLSSize)
3888 return nullptr;
3889 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3890 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3891 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3892 "_msarg_va_s");
3893 }
3894
3895 /// Compute the origin address for a given va_arg.
3896 Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) {
3897 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
3898 // getOriginPtrForVAArgument() is always called after
3899 // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
3900 // overflow.
3901 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3902 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
3903 "_msarg_va_o");
3904 }
3905
3906 void unpoisonVAListTagForInst(IntrinsicInst &I) {
3907 IRBuilder<> IRB(&I);
3908 Value *VAListTag = I.getArgOperand(0);
3909 Value *ShadowPtr, *OriginPtr;
3910 unsigned Alignment = 8;
3911 std::tie(ShadowPtr, OriginPtr) =
3912 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
3913 /*isStore*/ true);
3914
3915 // Unpoison the whole __va_list_tag.
3916 // FIXME: magic ABI constants.
3917 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3918 /* size */ 24, Alignment, false);
3919 // We shouldn't need to zero out the origins, as they're only checked for
3920 // nonzero shadow.
3921 }
3922
3923 void visitVAStartInst(VAStartInst &I) override {
3924 if (F.getCallingConv() == CallingConv::Win64)
3925 return;
3926 VAStartInstrumentationList.push_back(&I);
3927 unpoisonVAListTagForInst(I);
3928 }
3929
3930 void visitVACopyInst(VACopyInst &I) override {
3931 if (F.getCallingConv() == CallingConv::Win64) return;
3932 unpoisonVAListTagForInst(I);
3933 }
3934
3935 void finalizeInstrumentation() override {
3936 assert(!VAArgOverflowSize && !VAArgTLSCopy &&((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3937, __PRETTY_FUNCTION__))
3937 "finalizeInstrumentation called twice")((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3937, __PRETTY_FUNCTION__))
;
3938 if (!VAStartInstrumentationList.empty()) {
3939 // If there is a va_start in this function, make a backup copy of
3940 // va_arg_tls somewhere in the function entry block.
3941 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3942 VAArgOverflowSize =
3943 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
3944 Value *CopySize =
3945 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
3946 VAArgOverflowSize);
3947 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3948 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3949 if (MS.TrackOrigins) {
3950 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3951 IRB.CreateMemCpy(VAArgTLSOriginCopy, 8, MS.VAArgOriginTLS, 8, CopySize);
3952 }
3953 }
3954
3955 // Instrument va_start.
3956 // Copy va_list shadow from the backup copy of the TLS contents.
3957 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3958 CallInst *OrigInst = VAStartInstrumentationList[i];
3959 IRBuilder<> IRB(OrigInst->getNextNode());
3960 Value *VAListTag = OrigInst->getArgOperand(0);
3961
3962 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
3963 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
3964 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3965 ConstantInt::get(MS.IntptrTy, 16)),
3966 PointerType::get(RegSaveAreaPtrTy, 0));
3967 Value *RegSaveAreaPtr =
3968 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
3969 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
3970 unsigned Alignment = 16;
3971 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
3972 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3973 Alignment, /*isStore*/ true);
3974 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
3975 AMD64FpEndOffset);
3976 if (MS.TrackOrigins)
3977 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
3978 Alignment, AMD64FpEndOffset);
3979 Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
3980 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
3981 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3982 ConstantInt::get(MS.IntptrTy, 8)),
3983 PointerType::get(OverflowArgAreaPtrTy, 0));
3984 Value *OverflowArgAreaPtr =
3985 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
3986 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
3987 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
3988 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
3989 Alignment, /*isStore*/ true);
3990 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
3991 AMD64FpEndOffset);
3992 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
3993 VAArgOverflowSize);
3994 if (MS.TrackOrigins) {
3995 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
3996 AMD64FpEndOffset);
3997 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
3998 VAArgOverflowSize);
3999 }
4000 }
4001 }
4002};
4003
4004/// MIPS64-specific implementation of VarArgHelper.
4005struct VarArgMIPS64Helper : public VarArgHelper {
4006 Function &F;
4007 MemorySanitizer &MS;
4008 MemorySanitizerVisitor &MSV;
4009 Value *VAArgTLSCopy = nullptr;
4010 Value *VAArgSize = nullptr;
4011
4012 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4013
4014 VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
4015 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4016
4017 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
4018 unsigned VAArgOffset = 0;
4019 const DataLayout &DL = F.getParent()->getDataLayout();
4020 for (CallSite::arg_iterator ArgIt = CS.arg_begin() +
4021 CS.getFunctionType()->getNumParams(), End = CS.arg_end();
4022 ArgIt != End; ++ArgIt) {
4023 Triple TargetTriple(F.getParent()->getTargetTriple());
4024 Value *A = *ArgIt;
4025 Value *Base;
4026 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4027 if (TargetTriple.getArch() == Triple::mips64) {
4028 // Adjusting the shadow for argument with size < 8 to match the placement
4029 // of bits in big endian system
4030 if (ArgSize < 8)
4031 VAArgOffset += (8 - ArgSize);
4032 }
4033 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize);
4034 VAArgOffset += ArgSize;
4035 VAArgOffset = alignTo(VAArgOffset, 8);
4036 if (!Base)
4037 continue;
4038 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4039 }
4040
4041 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
4042 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4043 // a new class member i.e. it is the total size of all VarArgs.
4044 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4045 }
4046
4047 /// Compute the shadow address for a given va_arg.
4048 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4049 unsigned ArgOffset, unsigned ArgSize) {
4050 // Make sure we don't overflow __msan_va_arg_tls.
4051 if (ArgOffset + ArgSize > kParamTLSSize)
4052 return nullptr;
4053 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4054 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4055 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4056 "_msarg");
4057 }
4058
4059 void visitVAStartInst(VAStartInst &I) override {
4060 IRBuilder<> IRB(&I);
4061 VAStartInstrumentationList.push_back(&I);
4062 Value *VAListTag = I.getArgOperand(0);
4063 Value *ShadowPtr, *OriginPtr;
4064 unsigned Alignment = 8;
4065 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4066 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4067 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4068 /* size */ 8, Alignment, false);
4069 }
4070
4071 void visitVACopyInst(VACopyInst &I) override {
4072 IRBuilder<> IRB(&I);
4073 VAStartInstrumentationList.push_back(&I);
4074 Value *VAListTag = I.getArgOperand(0);
4075 Value *ShadowPtr, *OriginPtr;
4076 unsigned Alignment = 8;
4077 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4078 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4079 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4080 /* size */ 8, Alignment, false);
4081 }
4082
4083 void finalizeInstrumentation() override {
4084 assert(!VAArgSize && !VAArgTLSCopy &&((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4085, __PRETTY_FUNCTION__))
4085 "finalizeInstrumentation called twice")((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4085, __PRETTY_FUNCTION__))
;
4086 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
4087 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4088 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4089 VAArgSize);
4090
4091 if (!VAStartInstrumentationList.empty()) {
4092 // If there is a va_start in this function, make a backup copy of
4093 // va_arg_tls somewhere in the function entry block.
4094 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4095 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
4096 }
4097
4098 // Instrument va_start.
4099 // Copy va_list shadow from the backup copy of the TLS contents.
4100 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4101 CallInst *OrigInst = VAStartInstrumentationList[i];
4102 IRBuilder<> IRB(OrigInst->getNextNode());
4103 Value *VAListTag = OrigInst->getArgOperand(0);
4104 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4105 Value *RegSaveAreaPtrPtr =
4106 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4107 PointerType::get(RegSaveAreaPtrTy, 0));
4108 Value *RegSaveAreaPtr =
4109 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4110 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4111 unsigned Alignment = 8;
4112 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4113 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4114 Alignment, /*isStore*/ true);
4115 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4116 CopySize);
4117 }
4118 }
4119};
4120
4121/// AArch64-specific implementation of VarArgHelper.
4122struct VarArgAArch64Helper : public VarArgHelper {
4123 static const unsigned kAArch64GrArgSize = 64;
4124 static const unsigned kAArch64VrArgSize = 128;
4125
4126 static const unsigned AArch64GrBegOffset = 0;
4127 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
4128 // Make VR space aligned to 16 bytes.
4129 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
4130 static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
4131 + kAArch64VrArgSize;
4132 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
4133
4134 Function &F;
4135 MemorySanitizer &MS;
4136 MemorySanitizerVisitor &MSV;
4137 Value *VAArgTLSCopy = nullptr;
4138 Value *VAArgOverflowSize = nullptr;
4139
4140 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4141
4142 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4143
4144 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
4145 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4146
4147 ArgKind classifyArgument(Value* arg) {
4148 Type *T = arg->getType();
4149 if (T->isFPOrFPVectorTy())
4150 return AK_FloatingPoint;
4151 if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4152 || (T->isPointerTy()))
4153 return AK_GeneralPurpose;
4154 return AK_Memory;
4155 }
4156
4157 // The instrumentation stores the argument shadow in a non ABI-specific
4158 // format because it does not know which argument is named (since Clang,
4159 // like x86_64 case, lowers the va_args in the frontend and this pass only
4160 // sees the low level code that deals with va_list internals).
4161 // The first seven GR registers are saved in the first 56 bytes of the
4162 // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
4163 // the remaining arguments.
4164 // Using constant offset within the va_arg TLS array allows fast copy
4165 // in the finalize instrumentation.
4166 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
4167 unsigned GrOffset = AArch64GrBegOffset;
4168 unsigned VrOffset = AArch64VrBegOffset;
4169 unsigned OverflowOffset = AArch64VAEndOffset;
4170
4171 const DataLayout &DL = F.getParent()->getDataLayout();
4172 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
4173 ArgIt != End; ++ArgIt) {
4174 Value *A = *ArgIt;
4175 unsigned ArgNo = CS.getArgumentNo(ArgIt);
4176 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
4177 ArgKind AK = classifyArgument(A);
4178 if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
4179 AK = AK_Memory;
4180 if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
4181 AK = AK_Memory;
4182 Value *Base;
4183 switch (AK) {
4184 case AK_GeneralPurpose:
4185 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset, 8);
4186 GrOffset += 8;
4187 break;
4188 case AK_FloatingPoint:
4189 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset, 8);
4190 VrOffset += 16;
4191 break;
4192 case AK_Memory:
4193 // Don't count fixed arguments in the overflow area - va_start will
4194 // skip right over them.
4195 if (IsFixed)
4196 continue;
4197 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4198 Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset,
4199 alignTo(ArgSize, 8));
4200 OverflowOffset += alignTo(ArgSize, 8);
4201 break;
4202 }
4203 // Count Gp/Vr fixed arguments to their respective offsets, but don't
4204 // bother to actually store a shadow.
4205 if (IsFixed)
4206 continue;
4207 if (!Base)
4208 continue;
4209 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4210 }
4211 Constant *OverflowSize =
4212 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
4213 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4214 }
4215
4216 /// Compute the shadow address for a given va_arg.
4217 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4218 unsigned ArgOffset, unsigned ArgSize) {
4219 // Make sure we don't overflow __msan_va_arg_tls.
4220 if (ArgOffset + ArgSize > kParamTLSSize)
4221 return nullptr;
4222 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4223 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4224 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4225 "_msarg");
4226 }
4227
4228 void visitVAStartInst(VAStartInst &I) override {
4229 IRBuilder<> IRB(&I);
4230 VAStartInstrumentationList.push_back(&I);
4231 Value *VAListTag = I.getArgOperand(0);
4232 Value *ShadowPtr, *OriginPtr;
4233 unsigned Alignment = 8;
4234 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4235 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4236 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4237 /* size */ 32, Alignment, false);
4238 }
4239
4240 void visitVACopyInst(VACopyInst &I) override {
4241 IRBuilder<> IRB(&I);
4242 VAStartInstrumentationList.push_back(&I);
4243 Value *VAListTag = I.getArgOperand(0);
4244 Value *ShadowPtr, *OriginPtr;
4245 unsigned Alignment = 8;
4246 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4247 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4248 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4249 /* size */ 32, Alignment, false);
4250 }
4251
4252 // Retrieve a va_list field of 'void*' size.
4253 Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4254 Value *SaveAreaPtrPtr =
4255 IRB.CreateIntToPtr(
4256 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4257 ConstantInt::get(MS.IntptrTy, offset)),
4258 Type::getInt64PtrTy(*MS.C));
4259 return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
4260 }
4261
4262 // Retrieve a va_list field of 'int' size.
4263 Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4264 Value *SaveAreaPtr =
4265 IRB.CreateIntToPtr(
4266 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4267 ConstantInt::get(MS.IntptrTy, offset)),
4268 Type::getInt32PtrTy(*MS.C));
4269 Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
4270 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
4271 }
4272
4273 void finalizeInstrumentation() override {
4274 assert(!VAArgOverflowSize && !VAArgTLSCopy &&((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4275, __PRETTY_FUNCTION__))
4275 "finalizeInstrumentation called twice")((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4275, __PRETTY_FUNCTION__))
;
4276 if (!VAStartInstrumentationList.empty()) {
4277 // If there is a va_start in this function, make a backup copy of
4278 // va_arg_tls somewhere in the function entry block.
4279 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
4280 VAArgOverflowSize =
4281 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4282 Value *CopySize =
4283 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
4284 VAArgOverflowSize);
4285 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4286 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
4287 }
4288
4289 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
4290 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
4291
4292 // Instrument va_start, copy va_list shadow from the backup copy of
4293 // the TLS contents.
4294 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4295 CallInst *OrigInst = VAStartInstrumentationList[i];
4296 IRBuilder<> IRB(OrigInst->getNextNode());
4297
4298 Value *VAListTag = OrigInst->getArgOperand(0);
4299
4300 // The variadic ABI for AArch64 creates two areas to save the incoming
4301 // argument registers (one for 64-bit general register xn-x7 and another
4302 // for 128-bit FP/SIMD vn-v7).
4303 // We need then to propagate the shadow arguments on both regions
4304 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
4305 // The remaning arguments are saved on shadow for 'va::stack'.
4306 // One caveat is it requires only to propagate the non-named arguments,
4307 // however on the call site instrumentation 'all' the arguments are
4308 // saved. So to copy the shadow values from the va_arg TLS array
4309 // we need to adjust the offset for both GR and VR fields based on
4310 // the __{gr,vr}_offs value (since they are stores based on incoming
4311 // named arguments).
4312
4313 // Read the stack pointer from the va_list.
4314 Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
4315
4316 // Read both the __gr_top and __gr_off and add them up.
4317 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
4318 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
4319
4320 Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
4321
4322 // Read both the __vr_top and __vr_off and add them up.
4323 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
4324 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
4325
4326 Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
4327
4328 // It does not know how many named arguments is being used and, on the
4329 // callsite all the arguments were saved. Since __gr_off is defined as
4330 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
4331 // argument by ignoring the bytes of shadow from named arguments.
4332 Value *GrRegSaveAreaShadowPtrOff =
4333 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
4334
4335 Value *GrRegSaveAreaShadowPtr =
4336 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4337 /*Alignment*/ 8, /*isStore*/ true)
4338 .first;
4339
4340 Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4341 GrRegSaveAreaShadowPtrOff);
4342 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
4343
4344 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, 8, GrSrcPtr, 8, GrCopySize);
4345
4346 // Again, but for FP/SIMD values.
4347 Value *VrRegSaveAreaShadowPtrOff =
4348 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
4349
4350 Value *VrRegSaveAreaShadowPtr =
4351 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4352 /*Alignment*/ 8, /*isStore*/ true)
4353 .first;
4354
4355 Value *VrSrcPtr = IRB.CreateInBoundsGEP(
4356 IRB.getInt8Ty(),
4357 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4358 IRB.getInt32(AArch64VrBegOffset)),
4359 VrRegSaveAreaShadowPtrOff);
4360 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
4361
4362 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, 8, VrSrcPtr, 8, VrCopySize);
4363
4364 // And finally for remaining arguments.
4365 Value *StackSaveAreaShadowPtr =
4366 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
4367 /*Alignment*/ 16, /*isStore*/ true)
4368 .first;
4369
4370 Value *StackSrcPtr =
4371 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4372 IRB.getInt32(AArch64VAEndOffset));
4373
4374 IRB.CreateMemCpy(StackSaveAreaShadowPtr, 16, StackSrcPtr, 16,
4375 VAArgOverflowSize);
4376 }
4377 }
4378};
4379
4380/// PowerPC64-specific implementation of VarArgHelper.
4381struct VarArgPowerPC64Helper : public VarArgHelper {
4382 Function &F;
4383 MemorySanitizer &MS;
4384 MemorySanitizerVisitor &MSV;
4385 Value *VAArgTLSCopy = nullptr;
4386 Value *VAArgSize = nullptr;
4387
4388 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4389
4390 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
4391 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4392
4393 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
4394 // For PowerPC, we need to deal with alignment of stack arguments -
4395 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
4396 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
4397 // and QPX vectors are aligned to 32 bytes. For that reason, we
4398 // compute current offset from stack pointer (which is always properly
4399 // aligned), and offset for the first vararg, then subtract them.
4400 unsigned VAArgBase;
4401 Triple TargetTriple(F.getParent()->getTargetTriple());
4402 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
4403 // and 32 bytes for ABIv2. This is usually determined by target
4404 // endianness, but in theory could be overriden by function attribute.
4405 // For simplicity, we ignore it here (it'd only matter for QPX vectors).
4406 if (TargetTriple.getArch() == Triple::ppc64)
4407 VAArgBase = 48;
4408 else
4409 VAArgBase = 32;
4410 unsigned VAArgOffset = VAArgBase;
4411 const DataLayout &DL = F.getParent()->getDataLayout();
4412 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
4413 ArgIt != End; ++ArgIt) {
4414 Value *A = *ArgIt;
4415 unsigned ArgNo = CS.getArgumentNo(ArgIt);
4416 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
4417 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
4418 if (IsByVal) {
4419 assert(A->getType()->isPointerTy())((A->getType()->isPointerTy()) ? static_cast<void>
(0) : __assert_fail ("A->getType()->isPointerTy()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4419, __PRETTY_FUNCTION__))
;
4420 Type *RealTy = A->getType()->getPointerElementType();
4421 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4422 uint64_t ArgAlign = CS.getParamAlignment(ArgNo);
4423 if (ArgAlign < 8)
4424 ArgAlign = 8;
4425 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4426 if (!IsFixed) {
4427 Value *Base = getShadowPtrForVAArgument(
4428 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
4429 if (Base) {
4430 Value *AShadowPtr, *AOriginPtr;
4431 std::tie(AShadowPtr, AOriginPtr) =
4432 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
4433 kShadowTLSAlignment, /*isStore*/ false);
4434
4435 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
4436 kShadowTLSAlignment, ArgSize);
4437 }
4438 }
4439 VAArgOffset += alignTo(ArgSize, 8);
4440 } else {
4441 Value *Base;
4442 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4443 uint64_t ArgAlign = 8;
4444 if (A->getType()->isArrayTy()) {
4445 // Arrays are aligned to element size, except for long double
4446 // arrays, which are aligned to 8 bytes.
4447 Type *ElementTy = A->getType()->getArrayElementType();
4448 if (!ElementTy->isPPC_FP128Ty())
4449 ArgAlign = DL.getTypeAllocSize(ElementTy);
4450 } else if (A->getType()->isVectorTy()) {
4451 // Vectors are naturally aligned.
4452 ArgAlign = DL.getTypeAllocSize(A->getType());
4453 }
4454 if (ArgAlign < 8)
4455 ArgAlign = 8;
4456 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4457 if (DL.isBigEndian()) {
4458 // Adjusting the shadow for argument with size < 8 to match the placement
4459 // of bits in big endian system
4460 if (ArgSize < 8)
4461 VAArgOffset += (8 - ArgSize);
4462 }
4463 if (!IsFixed) {
4464 Base = getShadowPtrForVAArgument(A->getType(), IRB,
4465 VAArgOffset - VAArgBase, ArgSize);
4466 if (Base)
4467 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4468 }
4469 VAArgOffset += ArgSize;
4470 VAArgOffset = alignTo(VAArgOffset, 8);
4471 }
4472 if (IsFixed)
4473 VAArgBase = VAArgOffset;
4474 }
4475
4476 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
4477 VAArgOffset - VAArgBase);
4478 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4479 // a new class member i.e. it is the total size of all VarArgs.
4480 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4481 }
4482
4483 /// Compute the shadow address for a given va_arg.
4484 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4485 unsigned ArgOffset, unsigned ArgSize) {
4486 // Make sure we don't overflow __msan_va_arg_tls.
4487 if (ArgOffset + ArgSize > kParamTLSSize)
4488 return nullptr;
4489 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4490 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4491 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4492 "_msarg");
4493 }
4494
4495 void visitVAStartInst(VAStartInst &I) override {
4496 IRBuilder<> IRB(&I);
4497 VAStartInstrumentationList.push_back(&I);
4498 Value *VAListTag = I.getArgOperand(0);
4499 Value *ShadowPtr, *OriginPtr;
4500 unsigned Alignment = 8;
4501 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4502 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4503 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4504 /* size */ 8, Alignment, false);
4505 }
4506
4507 void visitVACopyInst(VACopyInst &I) override {
4508 IRBuilder<> IRB(&I);
4509 Value *VAListTag = I.getArgOperand(0);
4510 Value *ShadowPtr, *OriginPtr;
4511 unsigned Alignment = 8;
4512 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4513 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4514 // Unpoison the whole __va_list_tag.
4515 // FIXME: magic ABI constants.
4516 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4517 /* size */ 8, Alignment, false);
4518 }
4519
4520 void finalizeInstrumentation() override {
4521 assert(!VAArgSize && !VAArgTLSCopy &&((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4522, __PRETTY_FUNCTION__))
4522 "finalizeInstrumentation called twice")((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4522, __PRETTY_FUNCTION__))
;
4523 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
4524 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4525 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4526 VAArgSize);
4527
4528 if (!VAStartInstrumentationList.empty()) {
4529 // If there is a va_start in this function, make a backup copy of
4530 // va_arg_tls somewhere in the function entry block.
4531 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4532 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
4533 }
4534
4535 // Instrument va_start.
4536 // Copy va_list shadow from the backup copy of the TLS contents.
4537 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4538 CallInst *OrigInst = VAStartInstrumentationList[i];
4539 IRBuilder<> IRB(OrigInst->getNextNode());
4540 Value *VAListTag = OrigInst->getArgOperand(0);
4541 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4542 Value *RegSaveAreaPtrPtr =
4543 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4544 PointerType::get(RegSaveAreaPtrTy, 0));
4545 Value *RegSaveAreaPtr =
4546 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4547 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4548 unsigned Alignment = 8;
4549 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4550 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4551 Alignment, /*isStore*/ true);
4552 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4553 CopySize);
4554 }
4555 }
4556};
4557
4558/// A no-op implementation of VarArgHelper.
4559struct VarArgNoOpHelper : public VarArgHelper {
4560 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
4561 MemorySanitizerVisitor &MSV) {}
4562
4563 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {}
4564
4565 void visitVAStartInst(VAStartInst &I) override {}
4566
4567 void visitVACopyInst(VACopyInst &I) override {}
4568
4569 void finalizeInstrumentation() override {}
4570};
4571
4572} // end anonymous namespace
4573
4574static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
4575 MemorySanitizerVisitor &Visitor) {
4576 // VarArg handling is only implemented on AMD64. False positives are possible
4577 // on other platforms.
4578 Triple TargetTriple(Func.getParent()->getTargetTriple());
4579 if (TargetTriple.getArch() == Triple::x86_64)
4580 return new VarArgAMD64Helper(Func, Msan, Visitor);
4581 else if (TargetTriple.isMIPS64())
4582 return new VarArgMIPS64Helper(Func, Msan, Visitor);
4583 else if (TargetTriple.getArch() == Triple::aarch64)
4584 return new VarArgAArch64Helper(Func, Msan, Visitor);
4585 else if (TargetTriple.getArch() == Triple::ppc64 ||
4586 TargetTriple.getArch() == Triple::ppc64le)
4587 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
4588 else
4589 return new VarArgNoOpHelper(Func, Msan, Visitor);
4590}
4591
4592bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
4593 if (!CompileKernel
1.1
Field 'CompileKernel' is true
&& F.getName() == kMsanModuleCtorName)
2
Taking false branch
4594 return false;
4595
4596 MemorySanitizerVisitor Visitor(F, *this, TLI);
4597
4598 // Clear out readonly/readnone attributes.
4599 AttrBuilder B;
4600 B.addAttribute(Attribute::ReadOnly)
4601 .addAttribute(Attribute::ReadNone)
4602 .addAttribute(Attribute::WriteOnly)
4603 .addAttribute(Attribute::ArgMemOnly)
4604 .addAttribute(Attribute::Speculatable);
4605 F.removeAttributes(AttributeList::FunctionIndex, B);
4606
4607 return Visitor.runOnFunction();
3
Calling 'MemorySanitizerVisitor::runOnFunction'
4608}