Bug Summary

File:lib/Transforms/Instrumentation/MemorySanitizer.cpp
Warning:line 1795, column 17
2nd function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name MemorySanitizer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-9/lib/clang/9.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/include -I /build/llvm-toolchain-snapshot-9~svn362543/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/9.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-9/lib/clang/9.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/lib/Transforms/Instrumentation -fdebug-prefix-map=/build/llvm-toolchain-snapshot-9~svn362543=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2019-06-05-060531-1271-1 -x c++ /build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp -faddrsig
1//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of MemorySanitizer, a detector of uninitialized
11/// reads.
12///
13/// The algorithm of the tool is similar to Memcheck
14/// (http://goo.gl/QKbem). We associate a few shadow bits with every
15/// byte of the application memory, poison the shadow of the malloc-ed
16/// or alloca-ed memory, load the shadow bits on every memory read,
17/// propagate the shadow bits through some of the arithmetic
18/// instruction (including MOV), store the shadow bits on every memory
19/// write, report a bug on some other instructions (e.g. JMP) if the
20/// associated shadow is poisoned.
21///
22/// But there are differences too. The first and the major one:
23/// compiler instrumentation instead of binary instrumentation. This
24/// gives us much better register allocation, possible compiler
25/// optimizations and a fast start-up. But this brings the major issue
26/// as well: msan needs to see all program events, including system
27/// calls and reads/writes in system libraries, so we either need to
28/// compile *everything* with msan or use a binary translation
29/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30/// Another difference from Memcheck is that we use 8 shadow bits per
31/// byte of application memory and use a direct shadow mapping. This
32/// greatly simplifies the instrumentation code and avoids races on
33/// shadow updates (Memcheck is single-threaded so races are not a
34/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35/// path storage that uses 8 bits per byte).
36///
37/// The default value of shadow is 0, which means "clean" (not poisoned).
38///
39/// Every module initializer should call __msan_init to ensure that the
40/// shadow memory is ready. On error, __msan_warning is called. Since
41/// parameters and return values may be passed via registers, we have a
42/// specialized thread-local shadow for return values
43/// (__msan_retval_tls) and parameters (__msan_param_tls).
44///
45/// Origin tracking.
46///
47/// MemorySanitizer can track origins (allocation points) of all uninitialized
48/// values. This behavior is controlled with a flag (msan-track-origins) and is
49/// disabled by default.
50///
51/// Origins are 4-byte values created and interpreted by the runtime library.
52/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53/// of application memory. Propagation of origins is basically a bunch of
54/// "select" instructions that pick the origin of a dirty argument, if an
55/// instruction has one.
56///
57/// Every 4 aligned, consecutive bytes of application memory have one origin
58/// value associated with them. If these bytes contain uninitialized data
59/// coming from 2 different allocations, the last store wins. Because of this,
60/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61/// practice.
62///
63/// Origins are meaningless for fully initialized values, so MemorySanitizer
64/// avoids storing origin to memory when a fully initialized value is stored.
65/// This way it avoids needless overwritting origin of the 4-byte region on
66/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67///
68/// Atomic handling.
69///
70/// Ideally, every atomic store of application value should update the
71/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72/// of two disjoint locations can not be done without severe slowdown.
73///
74/// Therefore, we implement an approximation that may err on the safe side.
75/// In this implementation, every atomically accessed location in the program
76/// may only change from (partially) uninitialized to fully initialized, but
77/// not the other way around. We load the shadow _after_ the application load,
78/// and we store the shadow _before_ the app store. Also, we always store clean
79/// shadow (if the application store is atomic). This way, if the store-load
80/// pair constitutes a happens-before arc, shadow store and load are correctly
81/// ordered such that the load will get either the value that was stored, or
82/// some later value (which is always clean).
83///
84/// This does not work very well with Compare-And-Swap (CAS) and
85/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86/// must store the new shadow before the app operation, and load the shadow
87/// after the app operation. Computers don't work this way. Current
88/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89/// value. It implements the store part as a simple atomic store by storing a
90/// clean shadow.
91///
92/// Instrumenting inline assembly.
93///
94/// For inline assembly code LLVM has little idea about which memory locations
95/// become initialized depending on the arguments. It can be possible to figure
96/// out which arguments are meant to point to inputs and outputs, but the
97/// actual semantics can be only visible at runtime. In the Linux kernel it's
98/// also possible that the arguments only indicate the offset for a base taken
99/// from a segment register, so it's dangerous to treat any asm() arguments as
100/// pointers. We take a conservative approach generating calls to
101/// __msan_instrument_asm_store(ptr, size)
102/// , which defer the memory unpoisoning to the runtime library.
103/// The latter can perform more complex address checks to figure out whether
104/// it's safe to touch the shadow memory.
105/// Like with atomic operations, we call __msan_instrument_asm_store() before
106/// the assembly call, so that changes to the shadow memory will be seen by
107/// other threads together with main memory initialization.
108///
109/// KernelMemorySanitizer (KMSAN) implementation.
110///
111/// The major differences between KMSAN and MSan instrumentation are:
112/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113/// - KMSAN allocates shadow and origin memory for each page separately, so
114/// there are no explicit accesses to shadow and origin in the
115/// instrumentation.
116/// Shadow and origin values for a particular X-byte memory location
117/// (X=1,2,4,8) are accessed through pointers obtained via the
118/// __msan_metadata_ptr_for_load_X(ptr)
119/// __msan_metadata_ptr_for_store_X(ptr)
120/// functions. The corresponding functions check that the X-byte accesses
121/// are possible and returns the pointers to shadow and origin memory.
122/// Arbitrary sized accesses are handled with:
123/// __msan_metadata_ptr_for_load_n(ptr, size)
124/// __msan_metadata_ptr_for_store_n(ptr, size);
125/// - TLS variables are stored in a single per-task struct. A call to a
126/// function __msan_get_context_state() returning a pointer to that struct
127/// is inserted into every instrumented function before the entry block;
128/// - __msan_warning() takes a 32-bit origin parameter;
129/// - local variables are poisoned with __msan_poison_alloca() upon function
130/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
131/// function;
132/// - the pass doesn't declare any global variables or add global constructors
133/// to the translation unit.
134///
135/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
136/// calls, making sure we're on the safe side wrt. possible false positives.
137///
138/// KernelMemorySanitizer only supports X86_64 at the moment.
139///
140//===----------------------------------------------------------------------===//
141
142#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
143#include "llvm/ADT/APInt.h"
144#include "llvm/ADT/ArrayRef.h"
145#include "llvm/ADT/DepthFirstIterator.h"
146#include "llvm/ADT/SmallSet.h"
147#include "llvm/ADT/SmallString.h"
148#include "llvm/ADT/SmallVector.h"
149#include "llvm/ADT/StringExtras.h"
150#include "llvm/ADT/StringRef.h"
151#include "llvm/ADT/Triple.h"
152#include "llvm/Analysis/TargetLibraryInfo.h"
153#include "llvm/IR/Argument.h"
154#include "llvm/IR/Attributes.h"
155#include "llvm/IR/BasicBlock.h"
156#include "llvm/IR/CallSite.h"
157#include "llvm/IR/CallingConv.h"
158#include "llvm/IR/Constant.h"
159#include "llvm/IR/Constants.h"
160#include "llvm/IR/DataLayout.h"
161#include "llvm/IR/DerivedTypes.h"
162#include "llvm/IR/Function.h"
163#include "llvm/IR/GlobalValue.h"
164#include "llvm/IR/GlobalVariable.h"
165#include "llvm/IR/IRBuilder.h"
166#include "llvm/IR/InlineAsm.h"
167#include "llvm/IR/InstVisitor.h"
168#include "llvm/IR/InstrTypes.h"
169#include "llvm/IR/Instruction.h"
170#include "llvm/IR/Instructions.h"
171#include "llvm/IR/IntrinsicInst.h"
172#include "llvm/IR/Intrinsics.h"
173#include "llvm/IR/LLVMContext.h"
174#include "llvm/IR/MDBuilder.h"
175#include "llvm/IR/Module.h"
176#include "llvm/IR/Type.h"
177#include "llvm/IR/Value.h"
178#include "llvm/IR/ValueMap.h"
179#include "llvm/Pass.h"
180#include "llvm/Support/AtomicOrdering.h"
181#include "llvm/Support/Casting.h"
182#include "llvm/Support/CommandLine.h"
183#include "llvm/Support/Compiler.h"
184#include "llvm/Support/Debug.h"
185#include "llvm/Support/ErrorHandling.h"
186#include "llvm/Support/MathExtras.h"
187#include "llvm/Support/raw_ostream.h"
188#include "llvm/Transforms/Instrumentation.h"
189#include "llvm/Transforms/Utils/BasicBlockUtils.h"
190#include "llvm/Transforms/Utils/Local.h"
191#include "llvm/Transforms/Utils/ModuleUtils.h"
192#include <algorithm>
193#include <cassert>
194#include <cstddef>
195#include <cstdint>
196#include <memory>
197#include <string>
198#include <tuple>
199
200using namespace llvm;
201
202#define DEBUG_TYPE"msan" "msan"
203
204static const unsigned kOriginSize = 4;
205static const unsigned kMinOriginAlignment = 4;
206static const unsigned kShadowTLSAlignment = 8;
207
208// These constants must be kept in sync with the ones in msan.h.
209static const unsigned kParamTLSSize = 800;
210static const unsigned kRetvalTLSSize = 800;
211
212// Accesses sizes are powers of two: 1, 2, 4, 8.
213static const size_t kNumberOfAccessSizes = 4;
214
215/// Track origins of uninitialized values.
216///
217/// Adds a section to MemorySanitizer report that points to the allocation
218/// (stack or heap) the uninitialized bits came from originally.
219static cl::opt<int> ClTrackOrigins("msan-track-origins",
220 cl::desc("Track origins (allocation sites) of poisoned memory"),
221 cl::Hidden, cl::init(0));
222
223static cl::opt<bool> ClKeepGoing("msan-keep-going",
224 cl::desc("keep going after reporting a UMR"),
225 cl::Hidden, cl::init(false));
226
227static cl::opt<bool> ClPoisonStack("msan-poison-stack",
228 cl::desc("poison uninitialized stack variables"),
229 cl::Hidden, cl::init(true));
230
231static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
232 cl::desc("poison uninitialized stack variables with a call"),
233 cl::Hidden, cl::init(false));
234
235static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
236 cl::desc("poison uninitialized stack variables with the given pattern"),
237 cl::Hidden, cl::init(0xff));
238
239static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
240 cl::desc("poison undef temps"),
241 cl::Hidden, cl::init(true));
242
243static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
244 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
245 cl::Hidden, cl::init(true));
246
247static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
248 cl::desc("exact handling of relational integer ICmp"),
249 cl::Hidden, cl::init(false));
250
251static cl::opt<bool> ClHandleLifetimeIntrinsics(
252 "msan-handle-lifetime-intrinsics",
253 cl::desc(
254 "when possible, poison scoped variables at the beginning of the scope "
255 "(slower, but more precise)"),
256 cl::Hidden, cl::init(true));
257
258// When compiling the Linux kernel, we sometimes see false positives related to
259// MSan being unable to understand that inline assembly calls may initialize
260// local variables.
261// This flag makes the compiler conservatively unpoison every memory location
262// passed into an assembly call. Note that this may cause false positives.
263// Because it's impossible to figure out the array sizes, we can only unpoison
264// the first sizeof(type) bytes for each type* pointer.
265// The instrumentation is only enabled in KMSAN builds, and only if
266// -msan-handle-asm-conservative is on. This is done because we may want to
267// quickly disable assembly instrumentation when it breaks.
268static cl::opt<bool> ClHandleAsmConservative(
269 "msan-handle-asm-conservative",
270 cl::desc("conservative handling of inline assembly"), cl::Hidden,
271 cl::init(true));
272
273// This flag controls whether we check the shadow of the address
274// operand of load or store. Such bugs are very rare, since load from
275// a garbage address typically results in SEGV, but still happen
276// (e.g. only lower bits of address are garbage, or the access happens
277// early at program startup where malloc-ed memory is more likely to
278// be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
279static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
280 cl::desc("report accesses through a pointer which has poisoned shadow"),
281 cl::Hidden, cl::init(true));
282
283static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
284 cl::desc("print out instructions with default strict semantics"),
285 cl::Hidden, cl::init(false));
286
287static cl::opt<int> ClInstrumentationWithCallThreshold(
288 "msan-instrumentation-with-call-threshold",
289 cl::desc(
290 "If the function being instrumented requires more than "
291 "this number of checks and origin stores, use callbacks instead of "
292 "inline checks (-1 means never use callbacks)."),
293 cl::Hidden, cl::init(3500));
294
295static cl::opt<bool>
296 ClEnableKmsan("msan-kernel",
297 cl::desc("Enable KernelMemorySanitizer instrumentation"),
298 cl::Hidden, cl::init(false));
299
300// This is an experiment to enable handling of cases where shadow is a non-zero
301// compile-time constant. For some unexplainable reason they were silently
302// ignored in the instrumentation.
303static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
304 cl::desc("Insert checks for constant shadow values"),
305 cl::Hidden, cl::init(false));
306
307// This is off by default because of a bug in gold:
308// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
309static cl::opt<bool> ClWithComdat("msan-with-comdat",
310 cl::desc("Place MSan constructors in comdat sections"),
311 cl::Hidden, cl::init(false));
312
313// These options allow to specify custom memory map parameters
314// See MemoryMapParams for details.
315static cl::opt<uint64_t> ClAndMask("msan-and-mask",
316 cl::desc("Define custom MSan AndMask"),
317 cl::Hidden, cl::init(0));
318
319static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
320 cl::desc("Define custom MSan XorMask"),
321 cl::Hidden, cl::init(0));
322
323static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
324 cl::desc("Define custom MSan ShadowBase"),
325 cl::Hidden, cl::init(0));
326
327static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
328 cl::desc("Define custom MSan OriginBase"),
329 cl::Hidden, cl::init(0));
330
331static const char *const kMsanModuleCtorName = "msan.module_ctor";
332static const char *const kMsanInitName = "__msan_init";
333
334namespace {
335
336// Memory map parameters used in application-to-shadow address calculation.
337// Offset = (Addr & ~AndMask) ^ XorMask
338// Shadow = ShadowBase + Offset
339// Origin = OriginBase + Offset
340struct MemoryMapParams {
341 uint64_t AndMask;
342 uint64_t XorMask;
343 uint64_t ShadowBase;
344 uint64_t OriginBase;
345};
346
347struct PlatformMemoryMapParams {
348 const MemoryMapParams *bits32;
349 const MemoryMapParams *bits64;
350};
351
352} // end anonymous namespace
353
354// i386 Linux
355static const MemoryMapParams Linux_I386_MemoryMapParams = {
356 0x000080000000, // AndMask
357 0, // XorMask (not used)
358 0, // ShadowBase (not used)
359 0x000040000000, // OriginBase
360};
361
362// x86_64 Linux
363static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
364#ifdef MSAN_LINUX_X86_64_OLD_MAPPING
365 0x400000000000, // AndMask
366 0, // XorMask (not used)
367 0, // ShadowBase (not used)
368 0x200000000000, // OriginBase
369#else
370 0, // AndMask (not used)
371 0x500000000000, // XorMask
372 0, // ShadowBase (not used)
373 0x100000000000, // OriginBase
374#endif
375};
376
377// mips64 Linux
378static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
379 0, // AndMask (not used)
380 0x008000000000, // XorMask
381 0, // ShadowBase (not used)
382 0x002000000000, // OriginBase
383};
384
385// ppc64 Linux
386static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
387 0xE00000000000, // AndMask
388 0x100000000000, // XorMask
389 0x080000000000, // ShadowBase
390 0x1C0000000000, // OriginBase
391};
392
393// aarch64 Linux
394static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
395 0, // AndMask (not used)
396 0x06000000000, // XorMask
397 0, // ShadowBase (not used)
398 0x01000000000, // OriginBase
399};
400
401// i386 FreeBSD
402static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
403 0x000180000000, // AndMask
404 0x000040000000, // XorMask
405 0x000020000000, // ShadowBase
406 0x000700000000, // OriginBase
407};
408
409// x86_64 FreeBSD
410static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
411 0xc00000000000, // AndMask
412 0x200000000000, // XorMask
413 0x100000000000, // ShadowBase
414 0x380000000000, // OriginBase
415};
416
417// x86_64 NetBSD
418static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
419 0, // AndMask
420 0x500000000000, // XorMask
421 0, // ShadowBase
422 0x100000000000, // OriginBase
423};
424
425static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
426 &Linux_I386_MemoryMapParams,
427 &Linux_X86_64_MemoryMapParams,
428};
429
430static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
431 nullptr,
432 &Linux_MIPS64_MemoryMapParams,
433};
434
435static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
436 nullptr,
437 &Linux_PowerPC64_MemoryMapParams,
438};
439
440static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
441 nullptr,
442 &Linux_AArch64_MemoryMapParams,
443};
444
445static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
446 &FreeBSD_I386_MemoryMapParams,
447 &FreeBSD_X86_64_MemoryMapParams,
448};
449
450static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
451 nullptr,
452 &NetBSD_X86_64_MemoryMapParams,
453};
454
455namespace {
456
457/// Instrument functions of a module to detect uninitialized reads.
458///
459/// Instantiating MemorySanitizer inserts the msan runtime library API function
460/// declarations into the module if they don't exist already. Instantiating
461/// ensures the __msan_init function is in the list of global constructors for
462/// the module.
463class MemorySanitizer {
464public:
465 MemorySanitizer(Module &M, MemorySanitizerOptions Options) {
466 this->CompileKernel =
467 ClEnableKmsan.getNumOccurrences() > 0 ? ClEnableKmsan : Options.Kernel;
468 if (ClTrackOrigins.getNumOccurrences() > 0)
469 this->TrackOrigins = ClTrackOrigins;
470 else
471 this->TrackOrigins = this->CompileKernel ? 2 : Options.TrackOrigins;
472 this->Recover = ClKeepGoing.getNumOccurrences() > 0
473 ? ClKeepGoing
474 : (this->CompileKernel | Options.Recover);
475 initializeModule(M);
476 }
477
478 // MSan cannot be moved or copied because of MapParams.
479 MemorySanitizer(MemorySanitizer &&) = delete;
480 MemorySanitizer &operator=(MemorySanitizer &&) = delete;
481 MemorySanitizer(const MemorySanitizer &) = delete;
482 MemorySanitizer &operator=(const MemorySanitizer &) = delete;
483
484 bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
485
486private:
487 friend struct MemorySanitizerVisitor;
488 friend struct VarArgAMD64Helper;
489 friend struct VarArgMIPS64Helper;
490 friend struct VarArgAArch64Helper;
491 friend struct VarArgPowerPC64Helper;
492
493 void initializeModule(Module &M);
494 void initializeCallbacks(Module &M);
495 void createKernelApi(Module &M);
496 void createUserspaceApi(Module &M);
497
498 /// True if we're compiling the Linux kernel.
499 bool CompileKernel;
500 /// Track origins (allocation points) of uninitialized values.
501 int TrackOrigins;
502 bool Recover;
503
504 LLVMContext *C;
505 Type *IntptrTy;
506 Type *OriginTy;
507
508 // XxxTLS variables represent the per-thread state in MSan and per-task state
509 // in KMSAN.
510 // For the userspace these point to thread-local globals. In the kernel land
511 // they point to the members of a per-task struct obtained via a call to
512 // __msan_get_context_state().
513
514 /// Thread-local shadow storage for function parameters.
515 Value *ParamTLS;
516
517 /// Thread-local origin storage for function parameters.
518 Value *ParamOriginTLS;
519
520 /// Thread-local shadow storage for function return value.
521 Value *RetvalTLS;
522
523 /// Thread-local origin storage for function return value.
524 Value *RetvalOriginTLS;
525
526 /// Thread-local shadow storage for in-register va_arg function
527 /// parameters (x86_64-specific).
528 Value *VAArgTLS;
529
530 /// Thread-local shadow storage for in-register va_arg function
531 /// parameters (x86_64-specific).
532 Value *VAArgOriginTLS;
533
534 /// Thread-local shadow storage for va_arg overflow area
535 /// (x86_64-specific).
536 Value *VAArgOverflowSizeTLS;
537
538 /// Thread-local space used to pass origin value to the UMR reporting
539 /// function.
540 Value *OriginTLS;
541
542 /// Are the instrumentation callbacks set up?
543 bool CallbacksInitialized = false;
544
545 /// The run-time callback to print a warning.
546 FunctionCallee WarningFn;
547
548 // These arrays are indexed by log2(AccessSize).
549 FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
550 FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
551
552 /// Run-time helper that generates a new origin value for a stack
553 /// allocation.
554 FunctionCallee MsanSetAllocaOrigin4Fn;
555
556 /// Run-time helper that poisons stack on function entry.
557 FunctionCallee MsanPoisonStackFn;
558
559 /// Run-time helper that records a store (or any event) of an
560 /// uninitialized value and returns an updated origin id encoding this info.
561 FunctionCallee MsanChainOriginFn;
562
563 /// MSan runtime replacements for memmove, memcpy and memset.
564 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
565
566 /// KMSAN callback for task-local function argument shadow.
567 StructType *MsanContextStateTy;
568 FunctionCallee MsanGetContextStateFn;
569
570 /// Functions for poisoning/unpoisoning local variables
571 FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
572
573 /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
574 /// pointers.
575 FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
576 FunctionCallee MsanMetadataPtrForLoad_1_8[4];
577 FunctionCallee MsanMetadataPtrForStore_1_8[4];
578 FunctionCallee MsanInstrumentAsmStoreFn;
579
580 /// Helper to choose between different MsanMetadataPtrXxx().
581 FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
582
583 /// Memory map parameters used in application-to-shadow calculation.
584 const MemoryMapParams *MapParams;
585
586 /// Custom memory map parameters used when -msan-shadow-base or
587 // -msan-origin-base is provided.
588 MemoryMapParams CustomMapParams;
589
590 MDNode *ColdCallWeights;
591
592 /// Branch weights for origin store.
593 MDNode *OriginStoreWeights;
594
595 /// An empty volatile inline asm that prevents callback merge.
596 InlineAsm *EmptyAsm;
597
598 Function *MsanCtorFunction;
599};
600
601/// A legacy function pass for msan instrumentation.
602///
603/// Instruments functions to detect unitialized reads.
604struct MemorySanitizerLegacyPass : public FunctionPass {
605 // Pass identification, replacement for typeid.
606 static char ID;
607
608 MemorySanitizerLegacyPass(MemorySanitizerOptions Options = {})
609 : FunctionPass(ID), Options(Options) {}
610 StringRef getPassName() const override { return "MemorySanitizerLegacyPass"; }
611
612 void getAnalysisUsage(AnalysisUsage &AU) const override {
613 AU.addRequired<TargetLibraryInfoWrapperPass>();
614 }
615
616 bool runOnFunction(Function &F) override {
617 return MSan->sanitizeFunction(
618 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
619 }
620 bool doInitialization(Module &M) override;
621
622 Optional<MemorySanitizer> MSan;
623 MemorySanitizerOptions Options;
624};
625
626} // end anonymous namespace
627
628PreservedAnalyses MemorySanitizerPass::run(Function &F,
629 FunctionAnalysisManager &FAM) {
630 MemorySanitizer Msan(*F.getParent(), Options);
631 if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
632 return PreservedAnalyses::none();
633 return PreservedAnalyses::all();
634}
635
636char MemorySanitizerLegacyPass::ID = 0;
637
638INITIALIZE_PASS_BEGIN(MemorySanitizerLegacyPass, "msan",static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
639 "MemorySanitizer: detects uninitialized reads.", false,static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
640 false)static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
641INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
642INITIALIZE_PASS_END(MemorySanitizerLegacyPass, "msan",PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
643 "MemorySanitizer: detects uninitialized reads.", false,PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
644 false)PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
645
646FunctionPass *
647llvm::createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options) {
648 return new MemorySanitizerLegacyPass(Options);
649}
650
651/// Create a non-const global initialized with the given string.
652///
653/// Creates a writable global for Str so that we can pass it to the
654/// run-time lib. Runtime uses first 4 bytes of the string to store the
655/// frame ID, so the string needs to be mutable.
656static GlobalVariable *createPrivateNonConstGlobalForString(Module &M,
657 StringRef Str) {
658 Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
659 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
660 GlobalValue::PrivateLinkage, StrConst, "");
661}
662
663/// Create KMSAN API callbacks.
664void MemorySanitizer::createKernelApi(Module &M) {
665 IRBuilder<> IRB(*C);
666
667 // These will be initialized in insertKmsanPrologue().
668 RetvalTLS = nullptr;
669 RetvalOriginTLS = nullptr;
670 ParamTLS = nullptr;
671 ParamOriginTLS = nullptr;
672 VAArgTLS = nullptr;
673 VAArgOriginTLS = nullptr;
674 VAArgOverflowSizeTLS = nullptr;
675 // OriginTLS is unused in the kernel.
676 OriginTLS = nullptr;
677
678 // __msan_warning() in the kernel takes an origin.
679 WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(),
680 IRB.getInt32Ty());
681 // Requests the per-task context state (kmsan_context_state*) from the
682 // runtime library.
683 MsanContextStateTy = StructType::get(
684 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
685 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
686 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
687 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
688 IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
689 OriginTy);
690 MsanGetContextStateFn = M.getOrInsertFunction(
691 "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
692
693 Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
694 PointerType::get(IRB.getInt32Ty(), 0));
695
696 for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
697 std::string name_load =
698 "__msan_metadata_ptr_for_load_" + std::to_string(size);
699 std::string name_store =
700 "__msan_metadata_ptr_for_store_" + std::to_string(size);
701 MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
702 name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
703 MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
704 name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
705 }
706
707 MsanMetadataPtrForLoadN = M.getOrInsertFunction(
708 "__msan_metadata_ptr_for_load_n", RetTy,
709 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
710 MsanMetadataPtrForStoreN = M.getOrInsertFunction(
711 "__msan_metadata_ptr_for_store_n", RetTy,
712 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
713
714 // Functions for poisoning and unpoisoning memory.
715 MsanPoisonAllocaFn =
716 M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(),
717 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy());
718 MsanUnpoisonAllocaFn = M.getOrInsertFunction(
719 "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
720}
721
722static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
723 return M.getOrInsertGlobal(Name, Ty, [&] {
724 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
725 nullptr, Name, nullptr,
726 GlobalVariable::InitialExecTLSModel);
727 });
728}
729
730/// Insert declarations for userspace-specific functions and globals.
731void MemorySanitizer::createUserspaceApi(Module &M) {
732 IRBuilder<> IRB(*C);
733 // Create the callback.
734 // FIXME: this function should have "Cold" calling conv,
735 // which is not yet implemented.
736 StringRef WarningFnName = Recover ? "__msan_warning"
737 : "__msan_warning_noreturn";
738 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
739
740 // Create the global TLS variables.
741 RetvalTLS =
742 getOrInsertGlobal(M, "__msan_retval_tls",
743 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
744
745 RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy);
746
747 ParamTLS =
748 getOrInsertGlobal(M, "__msan_param_tls",
749 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
750
751 ParamOriginTLS =
752 getOrInsertGlobal(M, "__msan_param_origin_tls",
753 ArrayType::get(OriginTy, kParamTLSSize / 4));
754
755 VAArgTLS =
756 getOrInsertGlobal(M, "__msan_va_arg_tls",
757 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
758
759 VAArgOriginTLS =
760 getOrInsertGlobal(M, "__msan_va_arg_origin_tls",
761 ArrayType::get(OriginTy, kParamTLSSize / 4));
762
763 VAArgOverflowSizeTLS =
764 getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
765 OriginTLS = getOrInsertGlobal(M, "__msan_origin_tls", IRB.getInt32Ty());
766
767 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
768 AccessSizeIndex++) {
769 unsigned AccessSize = 1 << AccessSizeIndex;
770 std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
771 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
772 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
773 IRB.getInt32Ty());
774
775 FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
776 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
777 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
778 IRB.getInt8PtrTy(), IRB.getInt32Ty());
779 }
780
781 MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
782 "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
783 IRB.getInt8PtrTy(), IntptrTy);
784 MsanPoisonStackFn =
785 M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
786 IRB.getInt8PtrTy(), IntptrTy);
787}
788
789/// Insert extern declaration of runtime-provided functions and globals.
790void MemorySanitizer::initializeCallbacks(Module &M) {
791 // Only do this once.
792 if (CallbacksInitialized)
793 return;
794
795 IRBuilder<> IRB(*C);
796 // Initialize callbacks that are common for kernel and userspace
797 // instrumentation.
798 MsanChainOriginFn = M.getOrInsertFunction(
799 "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
800 MemmoveFn = M.getOrInsertFunction(
801 "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
802 IRB.getInt8PtrTy(), IntptrTy);
803 MemcpyFn = M.getOrInsertFunction(
804 "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
805 IntptrTy);
806 MemsetFn = M.getOrInsertFunction(
807 "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
808 IntptrTy);
809 // We insert an empty inline asm after __msan_report* to avoid callback merge.
810 EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
811 StringRef(""), StringRef(""),
812 /*hasSideEffects=*/true);
813
814 MsanInstrumentAsmStoreFn =
815 M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
816 PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
817
818 if (CompileKernel) {
819 createKernelApi(M);
820 } else {
821 createUserspaceApi(M);
822 }
823 CallbacksInitialized = true;
824}
825
826FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
827 int size) {
828 FunctionCallee *Fns =
829 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
830 switch (size) {
831 case 1:
832 return Fns[0];
833 case 2:
834 return Fns[1];
835 case 4:
836 return Fns[2];
837 case 8:
838 return Fns[3];
839 default:
840 return nullptr;
841 }
842}
843
844/// Module-level initialization.
845///
846/// inserts a call to __msan_init to the module's constructor list.
847void MemorySanitizer::initializeModule(Module &M) {
848 auto &DL = M.getDataLayout();
849
850 bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
851 bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
852 // Check the overrides first
853 if (ShadowPassed || OriginPassed) {
854 CustomMapParams.AndMask = ClAndMask;
855 CustomMapParams.XorMask = ClXorMask;
856 CustomMapParams.ShadowBase = ClShadowBase;
857 CustomMapParams.OriginBase = ClOriginBase;
858 MapParams = &CustomMapParams;
859 } else {
860 Triple TargetTriple(M.getTargetTriple());
861 switch (TargetTriple.getOS()) {
862 case Triple::FreeBSD:
863 switch (TargetTriple.getArch()) {
864 case Triple::x86_64:
865 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
866 break;
867 case Triple::x86:
868 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
869 break;
870 default:
871 report_fatal_error("unsupported architecture");
872 }
873 break;
874 case Triple::NetBSD:
875 switch (TargetTriple.getArch()) {
876 case Triple::x86_64:
877 MapParams = NetBSD_X86_MemoryMapParams.bits64;
878 break;
879 default:
880 report_fatal_error("unsupported architecture");
881 }
882 break;
883 case Triple::Linux:
884 switch (TargetTriple.getArch()) {
885 case Triple::x86_64:
886 MapParams = Linux_X86_MemoryMapParams.bits64;
887 break;
888 case Triple::x86:
889 MapParams = Linux_X86_MemoryMapParams.bits32;
890 break;
891 case Triple::mips64:
892 case Triple::mips64el:
893 MapParams = Linux_MIPS_MemoryMapParams.bits64;
894 break;
895 case Triple::ppc64:
896 case Triple::ppc64le:
897 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
898 break;
899 case Triple::aarch64:
900 case Triple::aarch64_be:
901 MapParams = Linux_ARM_MemoryMapParams.bits64;
902 break;
903 default:
904 report_fatal_error("unsupported architecture");
905 }
906 break;
907 default:
908 report_fatal_error("unsupported operating system");
909 }
910 }
911
912 C = &(M.getContext());
913 IRBuilder<> IRB(*C);
914 IntptrTy = IRB.getIntPtrTy(DL);
915 OriginTy = IRB.getInt32Ty();
916
917 ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
918 OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
919
920 if (!CompileKernel) {
921 std::tie(MsanCtorFunction, std::ignore) =
922 getOrCreateSanitizerCtorAndInitFunctions(
923 M, kMsanModuleCtorName, kMsanInitName,
924 /*InitArgTypes=*/{},
925 /*InitArgs=*/{},
926 // This callback is invoked when the functions are created the first
927 // time. Hook them into the global ctors list in that case:
928 [&](Function *Ctor, FunctionCallee) {
929 if (!ClWithComdat) {
930 appendToGlobalCtors(M, Ctor, 0);
931 return;
932 }
933 Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
934 Ctor->setComdat(MsanCtorComdat);
935 appendToGlobalCtors(M, Ctor, 0, Ctor);
936 });
937
938 if (TrackOrigins)
939 M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
940 return new GlobalVariable(
941 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
942 IRB.getInt32(TrackOrigins), "__msan_track_origins");
943 });
944
945 if (Recover)
946 M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
947 return new GlobalVariable(M, IRB.getInt32Ty(), true,
948 GlobalValue::WeakODRLinkage,
949 IRB.getInt32(Recover), "__msan_keep_going");
950 });
951}
952}
953
954bool MemorySanitizerLegacyPass::doInitialization(Module &M) {
955 MSan.emplace(M, Options);
956 return true;
957}
958
959namespace {
960
961/// A helper class that handles instrumentation of VarArg
962/// functions on a particular platform.
963///
964/// Implementations are expected to insert the instrumentation
965/// necessary to propagate argument shadow through VarArg function
966/// calls. Visit* methods are called during an InstVisitor pass over
967/// the function, and should avoid creating new basic blocks. A new
968/// instance of this class is created for each instrumented function.
969struct VarArgHelper {
970 virtual ~VarArgHelper() = default;
971
972 /// Visit a CallSite.
973 virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0;
974
975 /// Visit a va_start call.
976 virtual void visitVAStartInst(VAStartInst &I) = 0;
977
978 /// Visit a va_copy call.
979 virtual void visitVACopyInst(VACopyInst &I) = 0;
980
981 /// Finalize function instrumentation.
982 ///
983 /// This method is called after visiting all interesting (see above)
984 /// instructions in a function.
985 virtual void finalizeInstrumentation() = 0;
986};
987
988struct MemorySanitizerVisitor;
989
990} // end anonymous namespace
991
992static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
993 MemorySanitizerVisitor &Visitor);
994
995static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
996 if (TypeSize <= 8) return 0;
997 return Log2_32_Ceil((TypeSize + 7) / 8);
998}
999
1000namespace {
1001
1002/// This class does all the work for a given function. Store and Load
1003/// instructions store and load corresponding shadow and origin
1004/// values. Most instructions propagate shadow from arguments to their
1005/// return values. Certain instructions (most importantly, BranchInst)
1006/// test their argument shadow and print reports (with a runtime call) if it's
1007/// non-zero.
1008struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1009 Function &F;
1010 MemorySanitizer &MS;
1011 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1012 ValueMap<Value*, Value*> ShadowMap, OriginMap;
1013 std::unique_ptr<VarArgHelper> VAHelper;
1014 const TargetLibraryInfo *TLI;
1015 BasicBlock *ActualFnStart;
1016
1017 // The following flags disable parts of MSan instrumentation based on
1018 // blacklist contents and command-line options.
1019 bool InsertChecks;
1020 bool PropagateShadow;
1021 bool PoisonStack;
1022 bool PoisonUndef;
1023 bool CheckReturnValue;
1024
1025 struct ShadowOriginAndInsertPoint {
1026 Value *Shadow;
1027 Value *Origin;
1028 Instruction *OrigIns;
1029
1030 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1031 : Shadow(S), Origin(O), OrigIns(I) {}
1032 };
1033 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1034 bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics;
1035 SmallSet<AllocaInst *, 16> AllocaSet;
1036 SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1037 SmallVector<StoreInst *, 16> StoreList;
1038
1039 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1040 const TargetLibraryInfo &TLI)
1041 : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
1042 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
1043 InsertChecks = SanitizeFunction;
1044 PropagateShadow = SanitizeFunction;
1045 PoisonStack = SanitizeFunction && ClPoisonStack;
1046 PoisonUndef = SanitizeFunction && ClPoisonUndef;
1047 // FIXME: Consider using SpecialCaseList to specify a list of functions that
1048 // must always return fully initialized values. For now, we hardcode "main".
1049 CheckReturnValue = SanitizeFunction && (F.getName() == "main");
1050
1051 MS.initializeCallbacks(*F.getParent());
1052 if (MS.CompileKernel)
1053 ActualFnStart = insertKmsanPrologue(F);
1054 else
1055 ActualFnStart = &F.getEntryBlock();
1056
1057 LLVM_DEBUG(if (!InsertChecks) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
1058 << "MemorySanitizer is not inserting checks into '"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
1059 << F.getName() << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
;
1060 }
1061
1062 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1063 if (MS.TrackOrigins <= 1) return V;
1064 return IRB.CreateCall(MS.MsanChainOriginFn, V);
1065 }
1066
1067 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1068 const DataLayout &DL = F.getParent()->getDataLayout();
1069 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1070 if (IntptrSize == kOriginSize) return Origin;
1071 assert(IntptrSize == kOriginSize * 2)((IntptrSize == kOriginSize * 2) ? static_cast<void> (0
) : __assert_fail ("IntptrSize == kOriginSize * 2", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1071, __PRETTY_FUNCTION__))
;
1072 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
1073 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
1074 }
1075
1076 /// Fill memory range with the given origin value.
1077 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1078 unsigned Size, unsigned Alignment) {
1079 const DataLayout &DL = F.getParent()->getDataLayout();
1080 unsigned IntptrAlignment = DL.getABITypeAlignment(MS.IntptrTy);
1081 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1082 assert(IntptrAlignment >= kMinOriginAlignment)((IntptrAlignment >= kMinOriginAlignment) ? static_cast<
void> (0) : __assert_fail ("IntptrAlignment >= kMinOriginAlignment"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1082, __PRETTY_FUNCTION__))
;
1083 assert(IntptrSize >= kOriginSize)((IntptrSize >= kOriginSize) ? static_cast<void> (0)
: __assert_fail ("IntptrSize >= kOriginSize", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1083, __PRETTY_FUNCTION__))
;
1084
1085 unsigned Ofs = 0;
1086 unsigned CurrentAlignment = Alignment;
1087 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1088 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1089 Value *IntptrOriginPtr =
1090 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
1091 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1092 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
1093 : IntptrOriginPtr;
1094 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
1095 Ofs += IntptrSize / kOriginSize;
1096 CurrentAlignment = IntptrAlignment;
1097 }
1098 }
1099
1100 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1101 Value *GEP =
1102 i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1103 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
1104 CurrentAlignment = kMinOriginAlignment;
1105 }
1106 }
1107
1108 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1109 Value *OriginPtr, unsigned Alignment, bool AsCall) {
1110 const DataLayout &DL = F.getParent()->getDataLayout();
1111 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1112 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
1113 if (Shadow->getType()->isAggregateType()) {
1114 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1115 OriginAlignment);
1116 } else {
1117 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
1118 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
1119 if (ConstantShadow) {
1120 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
1121 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1122 OriginAlignment);
1123 return;
1124 }
1125
1126 unsigned TypeSizeInBits =
1127 DL.getTypeSizeInBits(ConvertedShadow->getType());
1128 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1129 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1130 FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1131 Value *ConvertedShadow2 = IRB.CreateZExt(
1132 ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1133 IRB.CreateCall(Fn, {ConvertedShadow2,
1134 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
1135 Origin});
1136 } else {
1137 Value *Cmp = IRB.CreateICmpNE(
1138 ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp");
1139 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1140 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
1141 IRBuilder<> IRBNew(CheckTerm);
1142 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1143 OriginAlignment);
1144 }
1145 }
1146 }
1147
1148 void materializeStores(bool InstrumentWithCalls) {
1149 for (StoreInst *SI : StoreList) {
1150 IRBuilder<> IRB(SI);
1151 Value *Val = SI->getValueOperand();
1152 Value *Addr = SI->getPointerOperand();
1153 Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1154 Value *ShadowPtr, *OriginPtr;
1155 Type *ShadowTy = Shadow->getType();
1156 unsigned Alignment = SI->getAlignment();
1157 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1158 std::tie(ShadowPtr, OriginPtr) =
1159 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1160
1161 StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
1162 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " STORE: " << *NewSI <<
"\n"; } } while (false)
;
1163 (void)NewSI;
1164
1165 if (SI->isAtomic())
1166 SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
1167
1168 if (MS.TrackOrigins && !SI->isAtomic())
1169 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1170 OriginAlignment, InstrumentWithCalls);
1171 }
1172 }
1173
1174 /// Helper function to insert a warning at IRB's current insert point.
1175 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1176 if (!Origin)
1177 Origin = (Value *)IRB.getInt32(0);
1178 if (MS.CompileKernel) {
1179 IRB.CreateCall(MS.WarningFn, Origin);
1180 } else {
1181 if (MS.TrackOrigins) {
1182 IRB.CreateStore(Origin, MS.OriginTLS);
1183 }
1184 IRB.CreateCall(MS.WarningFn, {});
1185 }
1186 IRB.CreateCall(MS.EmptyAsm, {});
1187 // FIXME: Insert UnreachableInst if !MS.Recover?
1188 // This may invalidate some of the following checks and needs to be done
1189 // at the very end.
1190 }
1191
1192 void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
1193 bool AsCall) {
1194 IRBuilder<> IRB(OrigIns);
1195 LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " SHAD0 : " << *Shadow <<
"\n"; } } while (false)
;
1196 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
1197 LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " SHAD1 : " << *ConvertedShadow
<< "\n"; } } while (false)
;
1198
1199 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
1200 if (ConstantShadow) {
1201 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
1202 insertWarningFn(IRB, Origin);
1203 }
1204 return;
1205 }
1206
1207 const DataLayout &DL = OrigIns->getModule()->getDataLayout();
1208
1209 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1210 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1211 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1212 FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1213 Value *ConvertedShadow2 =
1214 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1215 IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin
1216 ? Origin
1217 : (Value *)IRB.getInt32(0)});
1218 } else {
1219 Value *Cmp = IRB.CreateICmpNE(ConvertedShadow,
1220 getCleanShadow(ConvertedShadow), "_mscmp");
1221 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1222 Cmp, OrigIns,
1223 /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
1224
1225 IRB.SetInsertPoint(CheckTerm);
1226 insertWarningFn(IRB, Origin);
1227 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " CHECK: " << *Cmp <<
"\n"; } } while (false)
;
1228 }
1229 }
1230
1231 void materializeChecks(bool InstrumentWithCalls) {
1232 for (const auto &ShadowData : InstrumentationList) {
1233 Instruction *OrigIns = ShadowData.OrigIns;
1234 Value *Shadow = ShadowData.Shadow;
1235 Value *Origin = ShadowData.Origin;
1236 materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
1237 }
1238 LLVM_DEBUG(dbgs() << "DONE:\n" << F)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "DONE:\n" << F; } } while (
false)
;
1239 }
1240
1241 BasicBlock *insertKmsanPrologue(Function &F) {
1242 BasicBlock *ret =
1243 SplitBlock(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHI());
1244 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
1245 Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
1246 Constant *Zero = IRB.getInt32(0);
1247 MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1248 {Zero, IRB.getInt32(0)}, "param_shadow");
1249 MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1250 {Zero, IRB.getInt32(1)}, "retval_shadow");
1251 MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1252 {Zero, IRB.getInt32(2)}, "va_arg_shadow");
1253 MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1254 {Zero, IRB.getInt32(3)}, "va_arg_origin");
1255 MS.VAArgOverflowSizeTLS =
1256 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1257 {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
1258 MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1259 {Zero, IRB.getInt32(5)}, "param_origin");
1260 MS.RetvalOriginTLS =
1261 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1262 {Zero, IRB.getInt32(6)}, "retval_origin");
1263 return ret;
1264 }
1265
1266 /// Add MemorySanitizer instrumentation to a function.
1267 bool runOnFunction() {
1268 // In the presence of unreachable blocks, we may see Phi nodes with
1269 // incoming nodes from such blocks. Since InstVisitor skips unreachable
1270 // blocks, such nodes will not have any shadow value associated with them.
1271 // It's easier to remove unreachable blocks than deal with missing shadow.
1272 removeUnreachableBlocks(F);
1273
1274 // Iterate all BBs in depth-first order and create shadow instructions
1275 // for all instructions (where applicable).
1276 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1277 for (BasicBlock *BB : depth_first(ActualFnStart))
1278 visit(*BB);
1279
1280 // Finalize PHI nodes.
1281 for (PHINode *PN : ShadowPHINodes) {
1282 PHINode *PNS = cast<PHINode>(getShadow(PN));
1283 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1284 size_t NumValues = PN->getNumIncomingValues();
1285 for (size_t v = 0; v < NumValues; v++) {
1286 PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1287 if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1288 }
1289 }
1290
1291 VAHelper->finalizeInstrumentation();
1292
1293 // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1294 // instrumenting only allocas.
1295 if (InstrumentLifetimeStart) {
1296 for (auto Item : LifetimeStartList) {
1297 instrumentAlloca(*Item.second, Item.first);
1298 AllocaSet.erase(Item.second);
1299 }
1300 }
1301 // Poison the allocas for which we didn't instrument the corresponding
1302 // lifetime intrinsics.
1303 for (AllocaInst *AI : AllocaSet)
1304 instrumentAlloca(*AI);
1305
1306 bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
1307 InstrumentationList.size() + StoreList.size() >
1308 (unsigned)ClInstrumentationWithCallThreshold;
1309
1310 // Insert shadow value checks.
1311 materializeChecks(InstrumentWithCalls);
1312
1313 // Delayed instrumentation of StoreInst.
1314 // This may not add new address checks.
1315 materializeStores(InstrumentWithCalls);
1316
1317 return true;
1318 }
1319
1320 /// Compute the shadow type that corresponds to a given Value.
1321 Type *getShadowTy(Value *V) {
1322 return getShadowTy(V->getType());
1323 }
1324
1325 /// Compute the shadow type that corresponds to a given Type.
1326 Type *getShadowTy(Type *OrigTy) {
1327 if (!OrigTy->isSized()) {
1328 return nullptr;
1329 }
1330 // For integer type, shadow is the same as the original type.
1331 // This may return weird-sized types like i1.
1332 if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
1333 return IT;
1334 const DataLayout &DL = F.getParent()->getDataLayout();
1335 if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1336 uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1337 return VectorType::get(IntegerType::get(*MS.C, EltSize),
1338 VT->getNumElements());
1339 }
1340 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1341 return ArrayType::get(getShadowTy(AT->getElementType()),
1342 AT->getNumElements());
1343 }
1344 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1345 SmallVector<Type*, 4> Elements;
1346 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1347 Elements.push_back(getShadowTy(ST->getElementType(i)));
1348 StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
1349 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "getShadowTy: " << *ST <<
" ===> " << *Res << "\n"; } } while (false)
;
1350 return Res;
1351 }
1352 uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
1353 return IntegerType::get(*MS.C, TypeSize);
1354 }
1355
1356 /// Flatten a vector type.
1357 Type *getShadowTyNoVec(Type *ty) {
1358 if (VectorType *vt = dyn_cast<VectorType>(ty))
1359 return IntegerType::get(*MS.C, vt->getBitWidth());
1360 return ty;
1361 }
1362
1363 /// Convert a shadow value to it's flattened variant.
1364 Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) {
1365 Type *Ty = V->getType();
1366 Type *NoVecTy = getShadowTyNoVec(Ty);
1367 if (Ty == NoVecTy) return V;
1368 return IRB.CreateBitCast(V, NoVecTy);
1369 }
1370
1371 /// Compute the integer shadow offset that corresponds to a given
1372 /// application address.
1373 ///
1374 /// Offset = (Addr & ~AndMask) ^ XorMask
1375 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1376 Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
1377
1378 uint64_t AndMask = MS.MapParams->AndMask;
1379 if (AndMask)
1380 OffsetLong =
1381 IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
1382
1383 uint64_t XorMask = MS.MapParams->XorMask;
1384 if (XorMask)
1385 OffsetLong =
1386 IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
1387 return OffsetLong;
1388 }
1389
1390 /// Compute the shadow and origin addresses corresponding to a given
1391 /// application address.
1392 ///
1393 /// Shadow = ShadowBase + Offset
1394 /// Origin = (OriginBase + Offset) & ~3ULL
1395 std::pair<Value *, Value *> getShadowOriginPtrUserspace(Value *Addr,
1396 IRBuilder<> &IRB,
1397 Type *ShadowTy,
1398 unsigned Alignment) {
1399 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1400 Value *ShadowLong = ShadowOffset;
1401 uint64_t ShadowBase = MS.MapParams->ShadowBase;
1402 if (ShadowBase != 0) {
1403 ShadowLong =
1404 IRB.CreateAdd(ShadowLong,
1405 ConstantInt::get(MS.IntptrTy, ShadowBase));
1406 }
1407 Value *ShadowPtr =
1408 IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1409 Value *OriginPtr = nullptr;
1410 if (MS.TrackOrigins) {
1411 Value *OriginLong = ShadowOffset;
1412 uint64_t OriginBase = MS.MapParams->OriginBase;
1413 if (OriginBase != 0)
1414 OriginLong = IRB.CreateAdd(OriginLong,
1415 ConstantInt::get(MS.IntptrTy, OriginBase));
1416 if (Alignment < kMinOriginAlignment) {
1417 uint64_t Mask = kMinOriginAlignment - 1;
1418 OriginLong =
1419 IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
1420 }
1421 OriginPtr =
1422 IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0));
1423 }
1424 return std::make_pair(ShadowPtr, OriginPtr);
1425 }
1426
1427 std::pair<Value *, Value *>
1428 getShadowOriginPtrKernel(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1429 unsigned Alignment, bool isStore) {
1430 Value *ShadowOriginPtrs;
1431 const DataLayout &DL = F.getParent()->getDataLayout();
1432 int Size = DL.getTypeStoreSize(ShadowTy);
1433
1434 FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
1435 Value *AddrCast =
1436 IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
1437 if (Getter) {
1438 ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
1439 } else {
1440 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
1441 ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN
1442 : MS.MsanMetadataPtrForLoadN,
1443 {AddrCast, SizeVal});
1444 }
1445 Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
1446 ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
1447 Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
1448
1449 return std::make_pair(ShadowPtr, OriginPtr);
1450 }
1451
1452 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1453 Type *ShadowTy,
1454 unsigned Alignment,
1455 bool isStore) {
1456 std::pair<Value *, Value *> ret;
1457 if (MS.CompileKernel)
1458 ret = getShadowOriginPtrKernel(Addr, IRB, ShadowTy, Alignment, isStore);
1459 else
1460 ret = getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1461 return ret;
1462 }
1463
1464 /// Compute the shadow address for a given function argument.
1465 ///
1466 /// Shadow = ParamTLS+ArgOffset.
1467 Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
1468 int ArgOffset) {
1469 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1470 if (ArgOffset)
1471 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1472 return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1473 "_msarg");
1474 }
1475
1476 /// Compute the origin address for a given function argument.
1477 Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1478 int ArgOffset) {
1479 if (!MS.TrackOrigins)
1480 return nullptr;
1481 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1482 if (ArgOffset)
1483 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1484 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1485 "_msarg_o");
1486 }
1487
1488 /// Compute the shadow address for a retval.
1489 Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1490 return IRB.CreatePointerCast(MS.RetvalTLS,
1491 PointerType::get(getShadowTy(A), 0),
1492 "_msret");
1493 }
1494
1495 /// Compute the origin address for a retval.
1496 Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1497 // We keep a single origin for the entire retval. Might be too optimistic.
1498 return MS.RetvalOriginTLS;
1499 }
1500
1501 /// Set SV to be the shadow value for V.
1502 void setShadow(Value *V, Value *SV) {
1503 assert(!ShadowMap.count(V) && "Values may only have one shadow")((!ShadowMap.count(V) && "Values may only have one shadow"
) ? static_cast<void> (0) : __assert_fail ("!ShadowMap.count(V) && \"Values may only have one shadow\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1503, __PRETTY_FUNCTION__))
;
1504 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1505 }
1506
1507 /// Set Origin to be the origin value for V.
1508 void setOrigin(Value *V, Value *Origin) {
1509 if (!MS.TrackOrigins) return;
1510 assert(!OriginMap.count(V) && "Values may only have one origin")((!OriginMap.count(V) && "Values may only have one origin"
) ? static_cast<void> (0) : __assert_fail ("!OriginMap.count(V) && \"Values may only have one origin\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1510, __PRETTY_FUNCTION__))
;
1511 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "ORIGIN: " << *V << " ==> "
<< *Origin << "\n"; } } while (false)
;
1512 OriginMap[V] = Origin;
1513 }
1514
1515 Constant *getCleanShadow(Type *OrigTy) {
1516 Type *ShadowTy = getShadowTy(OrigTy);
1517 if (!ShadowTy)
1518 return nullptr;
1519 return Constant::getNullValue(ShadowTy);
1520 }
1521
1522 /// Create a clean shadow value for a given value.
1523 ///
1524 /// Clean shadow (all zeroes) means all bits of the value are defined
1525 /// (initialized).
1526 Constant *getCleanShadow(Value *V) {
1527 return getCleanShadow(V->getType());
1528 }
1529
1530 /// Create a dirty shadow of a given shadow type.
1531 Constant *getPoisonedShadow(Type *ShadowTy) {
1532 assert(ShadowTy)((ShadowTy) ? static_cast<void> (0) : __assert_fail ("ShadowTy"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1532, __PRETTY_FUNCTION__))
;
1533 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1534 return Constant::getAllOnesValue(ShadowTy);
1535 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1536 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1537 getPoisonedShadow(AT->getElementType()));
1538 return ConstantArray::get(AT, Vals);
1539 }
1540 if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1541 SmallVector<Constant *, 4> Vals;
1542 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1543 Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1544 return ConstantStruct::get(ST, Vals);
1545 }
1546 llvm_unreachable("Unexpected shadow type")::llvm::llvm_unreachable_internal("Unexpected shadow type", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1546)
;
1547 }
1548
1549 /// Create a dirty shadow for a given value.
1550 Constant *getPoisonedShadow(Value *V) {
1551 Type *ShadowTy = getShadowTy(V);
1552 if (!ShadowTy)
1553 return nullptr;
1554 return getPoisonedShadow(ShadowTy);
1555 }
1556
1557 /// Create a clean (zero) origin.
1558 Value *getCleanOrigin() {
1559 return Constant::getNullValue(MS.OriginTy);
1560 }
1561
1562 /// Get the shadow value for a given Value.
1563 ///
1564 /// This function either returns the value set earlier with setShadow,
1565 /// or extracts if from ParamTLS (for function arguments).
1566 Value *getShadow(Value *V) {
1567 if (!PropagateShadow) return getCleanShadow(V);
1568 if (Instruction *I = dyn_cast<Instruction>(V)) {
1569 if (I->getMetadata("nosanitize"))
1570 return getCleanShadow(V);
1571 // For instructions the shadow is already stored in the map.
1572 Value *Shadow = ShadowMap[V];
1573 if (!Shadow) {
1574 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "No shadow: " << *V <<
"\n" << *(I->getParent()); } } while (false)
;
1575 (void)I;
1576 assert(Shadow && "No shadow for a value")((Shadow && "No shadow for a value") ? static_cast<
void> (0) : __assert_fail ("Shadow && \"No shadow for a value\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1576, __PRETTY_FUNCTION__))
;
1577 }
1578 return Shadow;
1579 }
1580 if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1581 Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V);
1582 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Undef: " << *U << " ==> "
<< *AllOnes << "\n"; } } while (false)
;
1583 (void)U;
1584 return AllOnes;
1585 }
1586 if (Argument *A = dyn_cast<Argument>(V)) {
1587 // For arguments we compute the shadow on demand and store it in the map.
1588 Value **ShadowPtr = &ShadowMap[V];
1589 if (*ShadowPtr)
1590 return *ShadowPtr;
1591 Function *F = A->getParent();
1592 IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI());
1593 unsigned ArgOffset = 0;
1594 const DataLayout &DL = F->getParent()->getDataLayout();
1595 for (auto &FArg : F->args()) {
1596 if (!FArg.getType()->isSized()) {
1597 LLVM_DEBUG(dbgs() << "Arg is not sized\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Arg is not sized\n"; } } while (
false)
;
1598 continue;
1599 }
1600 unsigned Size =
1601 FArg.hasByValAttr()
1602 ? DL.getTypeAllocSize(FArg.getType()->getPointerElementType())
1603 : DL.getTypeAllocSize(FArg.getType());
1604 if (A == &FArg) {
1605 bool Overflow = ArgOffset + Size > kParamTLSSize;
1606 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1607 if (FArg.hasByValAttr()) {
1608 // ByVal pointer itself has clean shadow. We copy the actual
1609 // argument shadow to the underlying memory.
1610 // Figure out maximal valid memcpy alignment.
1611 unsigned ArgAlign = FArg.getParamAlignment();
1612 if (ArgAlign == 0) {
1613 Type *EltType = A->getType()->getPointerElementType();
1614 ArgAlign = DL.getABITypeAlignment(EltType);
1615 }
1616 Value *CpShadowPtr =
1617 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1618 /*isStore*/ true)
1619 .first;
1620 // TODO(glider): need to copy origins.
1621 if (Overflow) {
1622 // ParamTLS overflow.
1623 EntryIRB.CreateMemSet(
1624 CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
1625 Size, ArgAlign);
1626 } else {
1627 unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1628 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
1629 CopyAlign, Size);
1630 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ByValCpy: " << *Cpy <<
"\n"; } } while (false)
;
1631 (void)Cpy;
1632 }
1633 *ShadowPtr = getCleanShadow(V);
1634 } else {
1635 if (Overflow) {
1636 // ParamTLS overflow.
1637 *ShadowPtr = getCleanShadow(V);
1638 } else {
1639 *ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base,
1640 kShadowTLSAlignment);
1641 }
1642 }
1643 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ARG: " << FArg <<
" ==> " << **ShadowPtr << "\n"; } } while (false
)
1644 << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ARG: " << FArg <<
" ==> " << **ShadowPtr << "\n"; } } while (false
)
;
1645 if (MS.TrackOrigins && !Overflow) {
1646 Value *OriginPtr =
1647 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1648 setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
1649 } else {
1650 setOrigin(A, getCleanOrigin());
1651 }
1652 }
1653 ArgOffset += alignTo(Size, kShadowTLSAlignment);
1654 }
1655 assert(*ShadowPtr && "Could not find shadow for an argument")((*ShadowPtr && "Could not find shadow for an argument"
) ? static_cast<void> (0) : __assert_fail ("*ShadowPtr && \"Could not find shadow for an argument\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1655, __PRETTY_FUNCTION__))
;
1656 return *ShadowPtr;
1657 }
1658 // For everything else the shadow is zero.
1659 return getCleanShadow(V);
1660 }
1661
1662 /// Get the shadow for i-th argument of the instruction I.
1663 Value *getShadow(Instruction *I, int i) {
1664 return getShadow(I->getOperand(i));
1665 }
1666
1667 /// Get the origin for a value.
1668 Value *getOrigin(Value *V) {
1669 if (!MS.TrackOrigins) return nullptr;
1670 if (!PropagateShadow) return getCleanOrigin();
1671 if (isa<Constant>(V)) return getCleanOrigin();
1672 assert((isa<Instruction>(V) || isa<Argument>(V)) &&(((isa<Instruction>(V) || isa<Argument>(V)) &&
"Unexpected value type in getOrigin()") ? static_cast<void
> (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1673, __PRETTY_FUNCTION__))
1673 "Unexpected value type in getOrigin()")(((isa<Instruction>(V) || isa<Argument>(V)) &&
"Unexpected value type in getOrigin()") ? static_cast<void
> (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1673, __PRETTY_FUNCTION__))
;
1674 if (Instruction *I = dyn_cast<Instruction>(V)) {
1675 if (I->getMetadata("nosanitize"))
1676 return getCleanOrigin();
1677 }
1678 Value *Origin = OriginMap[V];
1679 assert(Origin && "Missing origin")((Origin && "Missing origin") ? static_cast<void>
(0) : __assert_fail ("Origin && \"Missing origin\"",
"/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1679, __PRETTY_FUNCTION__))
;
1680 return Origin;
1681 }
1682
1683 /// Get the origin for i-th argument of the instruction I.
1684 Value *getOrigin(Instruction *I, int i) {
1685 return getOrigin(I->getOperand(i));
1686 }
1687
1688 /// Remember the place where a shadow check should be inserted.
1689 ///
1690 /// This location will be later instrumented with a check that will print a
1691 /// UMR warning in runtime if the shadow value is not 0.
1692 void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1693 assert(Shadow)((Shadow) ? static_cast<void> (0) : __assert_fail ("Shadow"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1693, __PRETTY_FUNCTION__))
;
1694 if (!InsertChecks) return;
1695#ifndef NDEBUG
1696 Type *ShadowTy = Shadow->getType();
1697 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) &&(((isa<IntegerType>(ShadowTy) || isa<VectorType>(
ShadowTy)) && "Can only insert checks for integer and vector shadow types"
) ? static_cast<void> (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) && \"Can only insert checks for integer and vector shadow types\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1698, __PRETTY_FUNCTION__))
1698 "Can only insert checks for integer and vector shadow types")(((isa<IntegerType>(ShadowTy) || isa<VectorType>(
ShadowTy)) && "Can only insert checks for integer and vector shadow types"
) ? static_cast<void> (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) && \"Can only insert checks for integer and vector shadow types\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1698, __PRETTY_FUNCTION__))
;
1699#endif
1700 InstrumentationList.push_back(
1701 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1702 }
1703
1704 /// Remember the place where a shadow check should be inserted.
1705 ///
1706 /// This location will be later instrumented with a check that will print a
1707 /// UMR warning in runtime if the value is not fully defined.
1708 void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1709 assert(Val)((Val) ? static_cast<void> (0) : __assert_fail ("Val", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1709, __PRETTY_FUNCTION__))
;
1710 Value *Shadow, *Origin;
1711 if (ClCheckConstantShadow) {
1712 Shadow = getShadow(Val);
1713 if (!Shadow) return;
1714 Origin = getOrigin(Val);
1715 } else {
1716 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1717 if (!Shadow) return;
1718 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1719 }
1720 insertShadowCheck(Shadow, Origin, OrigIns);
1721 }
1722
1723 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
1724 switch (a) {
1725 case AtomicOrdering::NotAtomic:
1726 return AtomicOrdering::NotAtomic;
1727 case AtomicOrdering::Unordered:
1728 case AtomicOrdering::Monotonic:
1729 case AtomicOrdering::Release:
1730 return AtomicOrdering::Release;
1731 case AtomicOrdering::Acquire:
1732 case AtomicOrdering::AcquireRelease:
1733 return AtomicOrdering::AcquireRelease;
1734 case AtomicOrdering::SequentiallyConsistent:
1735 return AtomicOrdering::SequentiallyConsistent;
1736 }
1737 llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1737)
;
1738 }
1739
1740 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
1741 switch (a) {
1742 case AtomicOrdering::NotAtomic:
1743 return AtomicOrdering::NotAtomic;
1744 case AtomicOrdering::Unordered:
1745 case AtomicOrdering::Monotonic:
1746 case AtomicOrdering::Acquire:
1747 return AtomicOrdering::Acquire;
1748 case AtomicOrdering::Release:
1749 case AtomicOrdering::AcquireRelease:
1750 return AtomicOrdering::AcquireRelease;
1751 case AtomicOrdering::SequentiallyConsistent:
1752 return AtomicOrdering::SequentiallyConsistent;
1753 }
1754 llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1754)
;
1755 }
1756
1757 // ------------------- Visitors.
1758 using InstVisitor<MemorySanitizerVisitor>::visit;
1759 void visit(Instruction &I) {
1760 if (!I.getMetadata("nosanitize"))
1761 InstVisitor<MemorySanitizerVisitor>::visit(I);
1762 }
1763
1764 /// Instrument LoadInst
1765 ///
1766 /// Loads the corresponding shadow and (optionally) origin.
1767 /// Optionally, checks that the load address is fully defined.
1768 void visitLoadInst(LoadInst &I) {
1769 assert(I.getType()->isSized() && "Load type must have size")((I.getType()->isSized() && "Load type must have size"
) ? static_cast<void> (0) : __assert_fail ("I.getType()->isSized() && \"Load type must have size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1769, __PRETTY_FUNCTION__))
;
1
Assuming the condition is true
2
'?' condition is true
1770 assert(!I.getMetadata("nosanitize"))((!I.getMetadata("nosanitize")) ? static_cast<void> (0)
: __assert_fail ("!I.getMetadata(\"nosanitize\")", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1770, __PRETTY_FUNCTION__))
;
3
'?' condition is true
1771 IRBuilder<> IRB(I.getNextNode());
1772 Type *ShadowTy = getShadowTy(&I);
1773 Value *Addr = I.getPointerOperand();
1774 Value *ShadowPtr, *OriginPtr;
4
'OriginPtr' declared without an initial value
1775 unsigned Alignment = I.getAlignment();
1776 if (PropagateShadow) {
5
Assuming the condition is false
6
Taking false branch
1777 std::tie(ShadowPtr, OriginPtr) =
1778 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
1779 setShadow(&I,
1780 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
1781 } else {
1782 setShadow(&I, getCleanShadow(&I));
1783 }
1784
1785 if (ClCheckAccessAddress)
7
Assuming the condition is false
8
Taking false branch
1786 insertShadowCheck(I.getPointerOperand(), &I);
1787
1788 if (I.isAtomic())
9
Assuming the condition is false
10
Taking false branch
1789 I.setOrdering(addAcquireOrdering(I.getOrdering()));
1790
1791 if (MS.TrackOrigins) {
11
Assuming the condition is true
12
Taking true branch
1792 if (PropagateShadow) {
13
Assuming the condition is true
14
Taking true branch
1793 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1794 setOrigin(
1795 &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
15
2nd function call argument is an uninitialized value
1796 } else {
1797 setOrigin(&I, getCleanOrigin());
1798 }
1799 }
1800 }
1801
1802 /// Instrument StoreInst
1803 ///
1804 /// Stores the corresponding shadow and (optionally) origin.
1805 /// Optionally, checks that the store address is fully defined.
1806 void visitStoreInst(StoreInst &I) {
1807 StoreList.push_back(&I);
1808 if (ClCheckAccessAddress)
1809 insertShadowCheck(I.getPointerOperand(), &I);
1810 }
1811
1812 void handleCASOrRMW(Instruction &I) {
1813 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I))((isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>
(I)) ? static_cast<void> (0) : __assert_fail ("isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I)"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1813, __PRETTY_FUNCTION__))
;
1814
1815 IRBuilder<> IRB(&I);
1816 Value *Addr = I.getOperand(0);
1817 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, I.getType(),
1818 /*Alignment*/ 1, /*isStore*/ true)
1819 .first;
1820
1821 if (ClCheckAccessAddress)
1822 insertShadowCheck(Addr, &I);
1823
1824 // Only test the conditional argument of cmpxchg instruction.
1825 // The other argument can potentially be uninitialized, but we can not
1826 // detect this situation reliably without possible false positives.
1827 if (isa<AtomicCmpXchgInst>(I))
1828 insertShadowCheck(I.getOperand(1), &I);
1829
1830 IRB.CreateStore(getCleanShadow(&I), ShadowPtr);
1831
1832 setShadow(&I, getCleanShadow(&I));
1833 setOrigin(&I, getCleanOrigin());
1834 }
1835
1836 void visitAtomicRMWInst(AtomicRMWInst &I) {
1837 handleCASOrRMW(I);
1838 I.setOrdering(addReleaseOrdering(I.getOrdering()));
1839 }
1840
1841 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1842 handleCASOrRMW(I);
1843 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1844 }
1845
1846 // Vector manipulation.
1847 void visitExtractElementInst(ExtractElementInst &I) {
1848 insertShadowCheck(I.getOperand(1), &I);
1849 IRBuilder<> IRB(&I);
1850 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
1851 "_msprop"));
1852 setOrigin(&I, getOrigin(&I, 0));
1853 }
1854
1855 void visitInsertElementInst(InsertElementInst &I) {
1856 insertShadowCheck(I.getOperand(2), &I);
1857 IRBuilder<> IRB(&I);
1858 setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
1859 I.getOperand(2), "_msprop"));
1860 setOriginForNaryOp(I);
1861 }
1862
1863 void visitShuffleVectorInst(ShuffleVectorInst &I) {
1864 insertShadowCheck(I.getOperand(2), &I);
1865 IRBuilder<> IRB(&I);
1866 setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
1867 I.getOperand(2), "_msprop"));
1868 setOriginForNaryOp(I);
1869 }
1870
1871 // Casts.
1872 void visitSExtInst(SExtInst &I) {
1873 IRBuilder<> IRB(&I);
1874 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
1875 setOrigin(&I, getOrigin(&I, 0));
1876 }
1877
1878 void visitZExtInst(ZExtInst &I) {
1879 IRBuilder<> IRB(&I);
1880 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
1881 setOrigin(&I, getOrigin(&I, 0));
1882 }
1883
1884 void visitTruncInst(TruncInst &I) {
1885 IRBuilder<> IRB(&I);
1886 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
1887 setOrigin(&I, getOrigin(&I, 0));
1888 }
1889
1890 void visitBitCastInst(BitCastInst &I) {
1891 // Special case: if this is the bitcast (there is exactly 1 allowed) between
1892 // a musttail call and a ret, don't instrument. New instructions are not
1893 // allowed after a musttail call.
1894 if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
1895 if (CI->isMustTailCall())
1896 return;
1897 IRBuilder<> IRB(&I);
1898 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
1899 setOrigin(&I, getOrigin(&I, 0));
1900 }
1901
1902 void visitPtrToIntInst(PtrToIntInst &I) {
1903 IRBuilder<> IRB(&I);
1904 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1905 "_msprop_ptrtoint"));
1906 setOrigin(&I, getOrigin(&I, 0));
1907 }
1908
1909 void visitIntToPtrInst(IntToPtrInst &I) {
1910 IRBuilder<> IRB(&I);
1911 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1912 "_msprop_inttoptr"));
1913 setOrigin(&I, getOrigin(&I, 0));
1914 }
1915
1916 void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
1917 void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
1918 void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
1919 void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
1920 void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
1921 void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
1922
1923 /// Propagate shadow for bitwise AND.
1924 ///
1925 /// This code is exact, i.e. if, for example, a bit in the left argument
1926 /// is defined and 0, then neither the value not definedness of the
1927 /// corresponding bit in B don't affect the resulting shadow.
1928 void visitAnd(BinaryOperator &I) {
1929 IRBuilder<> IRB(&I);
1930 // "And" of 0 and a poisoned value results in unpoisoned value.
1931 // 1&1 => 1; 0&1 => 0; p&1 => p;
1932 // 1&0 => 0; 0&0 => 0; p&0 => 0;
1933 // 1&p => p; 0&p => 0; p&p => p;
1934 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
1935 Value *S1 = getShadow(&I, 0);
1936 Value *S2 = getShadow(&I, 1);
1937 Value *V1 = I.getOperand(0);
1938 Value *V2 = I.getOperand(1);
1939 if (V1->getType() != S1->getType()) {
1940 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1941 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1942 }
1943 Value *S1S2 = IRB.CreateAnd(S1, S2);
1944 Value *V1S2 = IRB.CreateAnd(V1, S2);
1945 Value *S1V2 = IRB.CreateAnd(S1, V2);
1946 setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1947 setOriginForNaryOp(I);
1948 }
1949
1950 void visitOr(BinaryOperator &I) {
1951 IRBuilder<> IRB(&I);
1952 // "Or" of 1 and a poisoned value results in unpoisoned value.
1953 // 1|1 => 1; 0|1 => 1; p|1 => 1;
1954 // 1|0 => 1; 0|0 => 0; p|0 => p;
1955 // 1|p => 1; 0|p => p; p|p => p;
1956 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
1957 Value *S1 = getShadow(&I, 0);
1958 Value *S2 = getShadow(&I, 1);
1959 Value *V1 = IRB.CreateNot(I.getOperand(0));
1960 Value *V2 = IRB.CreateNot(I.getOperand(1));
1961 if (V1->getType() != S1->getType()) {
1962 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1963 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1964 }
1965 Value *S1S2 = IRB.CreateAnd(S1, S2);
1966 Value *V1S2 = IRB.CreateAnd(V1, S2);
1967 Value *S1V2 = IRB.CreateAnd(S1, V2);
1968 setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1969 setOriginForNaryOp(I);
1970 }
1971
1972 /// Default propagation of shadow and/or origin.
1973 ///
1974 /// This class implements the general case of shadow propagation, used in all
1975 /// cases where we don't know and/or don't care about what the operation
1976 /// actually does. It converts all input shadow values to a common type
1977 /// (extending or truncating as necessary), and bitwise OR's them.
1978 ///
1979 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
1980 /// fully initialized), and less prone to false positives.
1981 ///
1982 /// This class also implements the general case of origin propagation. For a
1983 /// Nary operation, result origin is set to the origin of an argument that is
1984 /// not entirely initialized. If there is more than one such arguments, the
1985 /// rightmost of them is picked. It does not matter which one is picked if all
1986 /// arguments are initialized.
1987 template <bool CombineShadow>
1988 class Combiner {
1989 Value *Shadow = nullptr;
1990 Value *Origin = nullptr;
1991 IRBuilder<> &IRB;
1992 MemorySanitizerVisitor *MSV;
1993
1994 public:
1995 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
1996 : IRB(IRB), MSV(MSV) {}
1997
1998 /// Add a pair of shadow and origin values to the mix.
1999 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2000 if (CombineShadow) {
2001 assert(OpShadow)((OpShadow) ? static_cast<void> (0) : __assert_fail ("OpShadow"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2001, __PRETTY_FUNCTION__))
;
2002 if (!Shadow)
2003 Shadow = OpShadow;
2004 else {
2005 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2006 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
2007 }
2008 }
2009
2010 if (MSV->MS.TrackOrigins) {
2011 assert(OpOrigin)((OpOrigin) ? static_cast<void> (0) : __assert_fail ("OpOrigin"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2011, __PRETTY_FUNCTION__))
;
2012 if (!Origin) {
2013 Origin = OpOrigin;
2014 } else {
2015 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2016 // No point in adding something that might result in 0 origin value.
2017 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2018 Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
2019 Value *Cond =
2020 IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
2021 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2022 }
2023 }
2024 }
2025 return *this;
2026 }
2027
2028 /// Add an application value to the mix.
2029 Combiner &Add(Value *V) {
2030 Value *OpShadow = MSV->getShadow(V);
2031 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2032 return Add(OpShadow, OpOrigin);
2033 }
2034
2035 /// Set the current combined values as the given instruction's shadow
2036 /// and origin.
2037 void Done(Instruction *I) {
2038 if (CombineShadow) {
2039 assert(Shadow)((Shadow) ? static_cast<void> (0) : __assert_fail ("Shadow"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2039, __PRETTY_FUNCTION__))
;
2040 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
2041 MSV->setShadow(I, Shadow);
2042 }
2043 if (MSV->MS.TrackOrigins) {
2044 assert(Origin)((Origin) ? static_cast<void> (0) : __assert_fail ("Origin"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2044, __PRETTY_FUNCTION__))
;
2045 MSV->setOrigin(I, Origin);
2046 }
2047 }
2048 };
2049
2050 using ShadowAndOriginCombiner = Combiner<true>;
2051 using OriginCombiner = Combiner<false>;
2052
2053 /// Propagate origin for arbitrary operation.
2054 void setOriginForNaryOp(Instruction &I) {
2055 if (!MS.TrackOrigins) return;
2056 IRBuilder<> IRB(&I);
2057 OriginCombiner OC(this, IRB);
2058 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
2059 OC.Add(OI->get());
2060 OC.Done(&I);
2061 }
2062
2063 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2064 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&((!(Ty->isVectorTy() && Ty->getScalarType()->
isPointerTy()) && "Vector of pointers is not a valid shadow type"
) ? static_cast<void> (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2065, __PRETTY_FUNCTION__))
2065 "Vector of pointers is not a valid shadow type")((!(Ty->isVectorTy() && Ty->getScalarType()->
isPointerTy()) && "Vector of pointers is not a valid shadow type"
) ? static_cast<void> (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2065, __PRETTY_FUNCTION__))
;
2066 return Ty->isVectorTy() ?
2067 Ty->getVectorNumElements() * Ty->getScalarSizeInBits() :
2068 Ty->getPrimitiveSizeInBits();
2069 }
2070
2071 /// Cast between two shadow types, extending or truncating as
2072 /// necessary.
2073 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2074 bool Signed = false) {
2075 Type *srcTy = V->getType();
2076 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2077 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2078 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2079 return IRB.CreateICmpNE(V, getCleanShadow(V));
2080
2081 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2082 return IRB.CreateIntCast(V, dstTy, Signed);
2083 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2084 dstTy->getVectorNumElements() == srcTy->getVectorNumElements())
2085 return IRB.CreateIntCast(V, dstTy, Signed);
2086 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
2087 Value *V2 =
2088 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
2089 return IRB.CreateBitCast(V2, dstTy);
2090 // TODO: handle struct types.
2091 }
2092
2093 /// Cast an application value to the type of its own shadow.
2094 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2095 Type *ShadowTy = getShadowTy(V);
2096 if (V->getType() == ShadowTy)
2097 return V;
2098 if (V->getType()->isPtrOrPtrVectorTy())
2099 return IRB.CreatePtrToInt(V, ShadowTy);
2100 else
2101 return IRB.CreateBitCast(V, ShadowTy);
2102 }
2103
2104 /// Propagate shadow for arbitrary operation.
2105 void handleShadowOr(Instruction &I) {
2106 IRBuilder<> IRB(&I);
2107 ShadowAndOriginCombiner SC(this, IRB);
2108 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
2109 SC.Add(OI->get());
2110 SC.Done(&I);
2111 }
2112
2113 // Handle multiplication by constant.
2114 //
2115 // Handle a special case of multiplication by constant that may have one or
2116 // more zeros in the lower bits. This makes corresponding number of lower bits
2117 // of the result zero as well. We model it by shifting the other operand
2118 // shadow left by the required number of bits. Effectively, we transform
2119 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2120 // We use multiplication by 2**N instead of shift to cover the case of
2121 // multiplication by 0, which may occur in some elements of a vector operand.
2122 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2123 Value *OtherArg) {
2124 Constant *ShadowMul;
2125 Type *Ty = ConstArg->getType();
2126 if (Ty->isVectorTy()) {
2127 unsigned NumElements = Ty->getVectorNumElements();
2128 Type *EltTy = Ty->getSequentialElementType();
2129 SmallVector<Constant *, 16> Elements;
2130 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2131 if (ConstantInt *Elt =
2132 dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
2133 const APInt &V = Elt->getValue();
2134 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2135 Elements.push_back(ConstantInt::get(EltTy, V2));
2136 } else {
2137 Elements.push_back(ConstantInt::get(EltTy, 1));
2138 }
2139 }
2140 ShadowMul = ConstantVector::get(Elements);
2141 } else {
2142 if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2143 const APInt &V = Elt->getValue();
2144 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2145 ShadowMul = ConstantInt::get(Ty, V2);
2146 } else {
2147 ShadowMul = ConstantInt::get(Ty, 1);
2148 }
2149 }
2150
2151 IRBuilder<> IRB(&I);
2152 setShadow(&I,
2153 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
2154 setOrigin(&I, getOrigin(OtherArg));
2155 }
2156
2157 void visitMul(BinaryOperator &I) {
2158 Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
2159 Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
2160 if (constOp0 && !constOp1)
2161 handleMulByConstant(I, constOp0, I.getOperand(1));
2162 else if (constOp1 && !constOp0)
2163 handleMulByConstant(I, constOp1, I.getOperand(0));
2164 else
2165 handleShadowOr(I);
2166 }
2167
2168 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
2169 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
2170 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
2171 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
2172 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
2173 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
2174
2175 void handleIntegerDiv(Instruction &I) {
2176 IRBuilder<> IRB(&I);
2177 // Strict on the second argument.
2178 insertShadowCheck(I.getOperand(1), &I);
2179 setShadow(&I, getShadow(&I, 0));
2180 setOrigin(&I, getOrigin(&I, 0));
2181 }
2182
2183 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2184 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2185 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
2186 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
2187
2188 // Floating point division is side-effect free. We can not require that the
2189 // divisor is fully initialized and must propagate shadow. See PR37523.
2190 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
2191 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
2192
2193 /// Instrument == and != comparisons.
2194 ///
2195 /// Sometimes the comparison result is known even if some of the bits of the
2196 /// arguments are not.
2197 void handleEqualityComparison(ICmpInst &I) {
2198 IRBuilder<> IRB(&I);
2199 Value *A = I.getOperand(0);
2200 Value *B = I.getOperand(1);
2201 Value *Sa = getShadow(A);
2202 Value *Sb = getShadow(B);
2203
2204 // Get rid of pointers and vectors of pointers.
2205 // For ints (and vectors of ints), types of A and Sa match,
2206 // and this is a no-op.
2207 A = IRB.CreatePointerCast(A, Sa->getType());
2208 B = IRB.CreatePointerCast(B, Sb->getType());
2209
2210 // A == B <==> (C = A^B) == 0
2211 // A != B <==> (C = A^B) != 0
2212 // Sc = Sa | Sb
2213 Value *C = IRB.CreateXor(A, B);
2214 Value *Sc = IRB.CreateOr(Sa, Sb);
2215 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2216 // Result is defined if one of the following is true
2217 // * there is a defined 1 bit in C
2218 // * C is fully defined
2219 // Si = !(C & ~Sc) && Sc
2220 Value *Zero = Constant::getNullValue(Sc->getType());
2221 Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
2222 Value *Si =
2223 IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
2224 IRB.CreateICmpEQ(
2225 IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
2226 Si->setName("_msprop_icmp");
2227 setShadow(&I, Si);
2228 setOriginForNaryOp(I);
2229 }
2230
2231 /// Build the lowest possible value of V, taking into account V's
2232 /// uninitialized bits.
2233 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2234 bool isSigned) {
2235 if (isSigned) {
2236 // Split shadow into sign bit and other bits.
2237 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2238 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2239 // Maximise the undefined shadow bit, minimize other undefined bits.
2240 return
2241 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
2242 } else {
2243 // Minimize undefined bits.
2244 return IRB.CreateAnd(A, IRB.CreateNot(Sa));
2245 }
2246 }
2247
2248 /// Build the highest possible value of V, taking into account V's
2249 /// uninitialized bits.
2250 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2251 bool isSigned) {
2252 if (isSigned) {
2253 // Split shadow into sign bit and other bits.
2254 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2255 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2256 // Minimise the undefined shadow bit, maximise other undefined bits.
2257 return
2258 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
2259 } else {
2260 // Maximize undefined bits.
2261 return IRB.CreateOr(A, Sa);
2262 }
2263 }
2264
2265 /// Instrument relational comparisons.
2266 ///
2267 /// This function does exact shadow propagation for all relational
2268 /// comparisons of integers, pointers and vectors of those.
2269 /// FIXME: output seems suboptimal when one of the operands is a constant
2270 void handleRelationalComparisonExact(ICmpInst &I) {
2271 IRBuilder<> IRB(&I);
2272 Value *A = I.getOperand(0);
2273 Value *B = I.getOperand(1);
2274 Value *Sa = getShadow(A);
2275 Value *Sb = getShadow(B);
2276
2277 // Get rid of pointers and vectors of pointers.
2278 // For ints (and vectors of ints), types of A and Sa match,
2279 // and this is a no-op.
2280 A = IRB.CreatePointerCast(A, Sa->getType());
2281 B = IRB.CreatePointerCast(B, Sb->getType());
2282
2283 // Let [a0, a1] be the interval of possible values of A, taking into account
2284 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
2285 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
2286 bool IsSigned = I.isSigned();
2287 Value *S1 = IRB.CreateICmp(I.getPredicate(),
2288 getLowestPossibleValue(IRB, A, Sa, IsSigned),
2289 getHighestPossibleValue(IRB, B, Sb, IsSigned));
2290 Value *S2 = IRB.CreateICmp(I.getPredicate(),
2291 getHighestPossibleValue(IRB, A, Sa, IsSigned),
2292 getLowestPossibleValue(IRB, B, Sb, IsSigned));
2293 Value *Si = IRB.CreateXor(S1, S2);
2294 setShadow(&I, Si);
2295 setOriginForNaryOp(I);
2296 }
2297
2298 /// Instrument signed relational comparisons.
2299 ///
2300 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
2301 /// bit of the shadow. Everything else is delegated to handleShadowOr().
2302 void handleSignedRelationalComparison(ICmpInst &I) {
2303 Constant *constOp;
2304 Value *op = nullptr;
2305 CmpInst::Predicate pre;
2306 if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
2307 op = I.getOperand(0);
2308 pre = I.getPredicate();
2309 } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
2310 op = I.getOperand(1);
2311 pre = I.getSwappedPredicate();
2312 } else {
2313 handleShadowOr(I);
2314 return;
2315 }
2316
2317 if ((constOp->isNullValue() &&
2318 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2319 (constOp->isAllOnesValue() &&
2320 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2321 IRBuilder<> IRB(&I);
2322 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2323 "_msprop_icmp_s");
2324 setShadow(&I, Shadow);
2325 setOrigin(&I, getOrigin(op));
2326 } else {
2327 handleShadowOr(I);
2328 }
2329 }
2330
2331 void visitICmpInst(ICmpInst &I) {
2332 if (!ClHandleICmp) {
2333 handleShadowOr(I);
2334 return;
2335 }
2336 if (I.isEquality()) {
2337 handleEqualityComparison(I);
2338 return;
2339 }
2340
2341 assert(I.isRelational())((I.isRelational()) ? static_cast<void> (0) : __assert_fail
("I.isRelational()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2341, __PRETTY_FUNCTION__))
;
2342 if (ClHandleICmpExact) {
2343 handleRelationalComparisonExact(I);
2344 return;
2345 }
2346 if (I.isSigned()) {
2347 handleSignedRelationalComparison(I);
2348 return;
2349 }
2350
2351 assert(I.isUnsigned())((I.isUnsigned()) ? static_cast<void> (0) : __assert_fail
("I.isUnsigned()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2351, __PRETTY_FUNCTION__))
;
2352 if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
2353 handleRelationalComparisonExact(I);
2354 return;
2355 }
2356
2357 handleShadowOr(I);
2358 }
2359
2360 void visitFCmpInst(FCmpInst &I) {
2361 handleShadowOr(I);
2362 }
2363
2364 void handleShift(BinaryOperator &I) {
2365 IRBuilder<> IRB(&I);
2366 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2367 // Otherwise perform the same shift on S1.
2368 Value *S1 = getShadow(&I, 0);
2369 Value *S2 = getShadow(&I, 1);
2370 Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
2371 S2->getType());
2372 Value *V2 = I.getOperand(1);
2373 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2374 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2375 setOriginForNaryOp(I);
2376 }
2377
2378 void visitShl(BinaryOperator &I) { handleShift(I); }
2379 void visitAShr(BinaryOperator &I) { handleShift(I); }
2380 void visitLShr(BinaryOperator &I) { handleShift(I); }
2381
2382 /// Instrument llvm.memmove
2383 ///
2384 /// At this point we don't know if llvm.memmove will be inlined or not.
2385 /// If we don't instrument it and it gets inlined,
2386 /// our interceptor will not kick in and we will lose the memmove.
2387 /// If we instrument the call here, but it does not get inlined,
2388 /// we will memove the shadow twice: which is bad in case
2389 /// of overlapping regions. So, we simply lower the intrinsic to a call.
2390 ///
2391 /// Similar situation exists for memcpy and memset.
2392 void visitMemMoveInst(MemMoveInst &I) {
2393 IRBuilder<> IRB(&I);
2394 IRB.CreateCall(
2395 MS.MemmoveFn,
2396 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2397 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2398 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2399 I.eraseFromParent();
2400 }
2401
2402 // Similar to memmove: avoid copying shadow twice.
2403 // This is somewhat unfortunate as it may slowdown small constant memcpys.
2404 // FIXME: consider doing manual inline for small constant sizes and proper
2405 // alignment.
2406 void visitMemCpyInst(MemCpyInst &I) {
2407 IRBuilder<> IRB(&I);
2408 IRB.CreateCall(
2409 MS.MemcpyFn,
2410 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2411 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2412 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2413 I.eraseFromParent();
2414 }
2415
2416 // Same as memcpy.
2417 void visitMemSetInst(MemSetInst &I) {
2418 IRBuilder<> IRB(&I);
2419 IRB.CreateCall(
2420 MS.MemsetFn,
2421 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2422 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2423 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2424 I.eraseFromParent();
2425 }
2426
2427 void visitVAStartInst(VAStartInst &I) {
2428 VAHelper->visitVAStartInst(I);
2429 }
2430
2431 void visitVACopyInst(VACopyInst &I) {
2432 VAHelper->visitVACopyInst(I);
2433 }
2434
2435 /// Handle vector store-like intrinsics.
2436 ///
2437 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
2438 /// has 1 pointer argument and 1 vector argument, returns void.
2439 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2440 IRBuilder<> IRB(&I);
2441 Value* Addr = I.getArgOperand(0);
2442 Value *Shadow = getShadow(&I, 1);
2443 Value *ShadowPtr, *OriginPtr;
2444
2445 // We don't know the pointer alignment (could be unaligned SSE store!).
2446 // Have to assume to worst case.
2447 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2448 Addr, IRB, Shadow->getType(), /*Alignment*/ 1, /*isStore*/ true);
2449 IRB.CreateAlignedStore(Shadow, ShadowPtr, 1);
2450
2451 if (ClCheckAccessAddress)
2452 insertShadowCheck(Addr, &I);
2453
2454 // FIXME: factor out common code from materializeStores
2455 if (MS.TrackOrigins) IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2456 return true;
2457 }
2458
2459 /// Handle vector load-like intrinsics.
2460 ///
2461 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
2462 /// has 1 pointer argument, returns a vector.
2463 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2464 IRBuilder<> IRB(&I);
2465 Value *Addr = I.getArgOperand(0);
2466
2467 Type *ShadowTy = getShadowTy(&I);
2468 Value *ShadowPtr, *OriginPtr;
2469 if (PropagateShadow) {
2470 // We don't know the pointer alignment (could be unaligned SSE load!).
2471 // Have to assume to worst case.
2472 unsigned Alignment = 1;
2473 std::tie(ShadowPtr, OriginPtr) =
2474 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2475 setShadow(&I,
2476 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2477 } else {
2478 setShadow(&I, getCleanShadow(&I));
2479 }
2480
2481 if (ClCheckAccessAddress)
2482 insertShadowCheck(Addr, &I);
2483
2484 if (MS.TrackOrigins) {
2485 if (PropagateShadow)
2486 setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
2487 else
2488 setOrigin(&I, getCleanOrigin());
2489 }
2490 return true;
2491 }
2492
2493 /// Handle (SIMD arithmetic)-like intrinsics.
2494 ///
2495 /// Instrument intrinsics with any number of arguments of the same type,
2496 /// equal to the return type. The type should be simple (no aggregates or
2497 /// pointers; vectors are fine).
2498 /// Caller guarantees that this intrinsic does not access memory.
2499 bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2500 Type *RetTy = I.getType();
2501 if (!(RetTy->isIntOrIntVectorTy() ||
2502 RetTy->isFPOrFPVectorTy() ||
2503 RetTy->isX86_MMXTy()))
2504 return false;
2505
2506 unsigned NumArgOperands = I.getNumArgOperands();
2507
2508 for (unsigned i = 0; i < NumArgOperands; ++i) {
2509 Type *Ty = I.getArgOperand(i)->getType();
2510 if (Ty != RetTy)
2511 return false;
2512 }
2513
2514 IRBuilder<> IRB(&I);
2515 ShadowAndOriginCombiner SC(this, IRB);
2516 for (unsigned i = 0; i < NumArgOperands; ++i)
2517 SC.Add(I.getArgOperand(i));
2518 SC.Done(&I);
2519
2520 return true;
2521 }
2522
2523 /// Heuristically instrument unknown intrinsics.
2524 ///
2525 /// The main purpose of this code is to do something reasonable with all
2526 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2527 /// We recognize several classes of intrinsics by their argument types and
2528 /// ModRefBehaviour and apply special intrumentation when we are reasonably
2529 /// sure that we know what the intrinsic does.
2530 ///
2531 /// We special-case intrinsics where this approach fails. See llvm.bswap
2532 /// handling as an example of that.
2533 bool handleUnknownIntrinsic(IntrinsicInst &I) {
2534 unsigned NumArgOperands = I.getNumArgOperands();
2535 if (NumArgOperands == 0)
2536 return false;
2537
2538 if (NumArgOperands == 2 &&
2539 I.getArgOperand(0)->getType()->isPointerTy() &&
2540 I.getArgOperand(1)->getType()->isVectorTy() &&
2541 I.getType()->isVoidTy() &&
2542 !I.onlyReadsMemory()) {
2543 // This looks like a vector store.
2544 return handleVectorStoreIntrinsic(I);
2545 }
2546
2547 if (NumArgOperands == 1 &&
2548 I.getArgOperand(0)->getType()->isPointerTy() &&
2549 I.getType()->isVectorTy() &&
2550 I.onlyReadsMemory()) {
2551 // This looks like a vector load.
2552 return handleVectorLoadIntrinsic(I);
2553 }
2554
2555 if (I.doesNotAccessMemory())
2556 if (maybeHandleSimpleNomemIntrinsic(I))
2557 return true;
2558
2559 // FIXME: detect and handle SSE maskstore/maskload
2560 return false;
2561 }
2562
2563 void handleLifetimeStart(IntrinsicInst &I) {
2564 if (!PoisonStack)
2565 return;
2566 DenseMap<Value *, AllocaInst *> AllocaForValue;
2567 AllocaInst *AI =
2568 llvm::findAllocaForValue(I.getArgOperand(1), AllocaForValue);
2569 if (!AI)
2570 InstrumentLifetimeStart = false;
2571 LifetimeStartList.push_back(std::make_pair(&I, AI));
2572 }
2573
2574 void handleBswap(IntrinsicInst &I) {
2575 IRBuilder<> IRB(&I);
2576 Value *Op = I.getArgOperand(0);
2577 Type *OpType = Op->getType();
2578 Function *BswapFunc = Intrinsic::getDeclaration(
2579 F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2580 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2581 setOrigin(&I, getOrigin(Op));
2582 }
2583
2584 // Instrument vector convert instrinsic.
2585 //
2586 // This function instruments intrinsics like cvtsi2ss:
2587 // %Out = int_xxx_cvtyyy(%ConvertOp)
2588 // or
2589 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2590 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2591 // number \p Out elements, and (if has 2 arguments) copies the rest of the
2592 // elements from \p CopyOp.
2593 // In most cases conversion involves floating-point value which may trigger a
2594 // hardware exception when not fully initialized. For this reason we require
2595 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2596 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2597 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2598 // return a fully initialized value.
2599 void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) {
2600 IRBuilder<> IRB(&I);
2601 Value *CopyOp, *ConvertOp;
2602
2603 switch (I.getNumArgOperands()) {
2604 case 3:
2605 assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode")((isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantInt>(I.getArgOperand(2)) && \"Invalid rounding mode\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2605, __PRETTY_FUNCTION__))
;
2606 LLVM_FALLTHROUGH[[clang::fallthrough]];
2607 case 2:
2608 CopyOp = I.getArgOperand(0);
2609 ConvertOp = I.getArgOperand(1);
2610 break;
2611 case 1:
2612 ConvertOp = I.getArgOperand(0);
2613 CopyOp = nullptr;
2614 break;
2615 default:
2616 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.")::llvm::llvm_unreachable_internal("Cvt intrinsic with unsupported number of arguments."
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2616)
;
2617 }
2618
2619 // The first *NumUsedElements* elements of ConvertOp are converted to the
2620 // same number of output elements. The rest of the output is copied from
2621 // CopyOp, or (if not available) filled with zeroes.
2622 // Combine shadow for elements of ConvertOp that are used in this operation,
2623 // and insert a check.
2624 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2625 // int->any conversion.
2626 Value *ConvertShadow = getShadow(ConvertOp);
2627 Value *AggShadow = nullptr;
2628 if (ConvertOp->getType()->isVectorTy()) {
2629 AggShadow = IRB.CreateExtractElement(
2630 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2631 for (int i = 1; i < NumUsedElements; ++i) {
2632 Value *MoreShadow = IRB.CreateExtractElement(
2633 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2634 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2635 }
2636 } else {
2637 AggShadow = ConvertShadow;
2638 }
2639 assert(AggShadow->getType()->isIntegerTy())((AggShadow->getType()->isIntegerTy()) ? static_cast<
void> (0) : __assert_fail ("AggShadow->getType()->isIntegerTy()"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2639, __PRETTY_FUNCTION__))
;
2640 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2641
2642 // Build result shadow by zero-filling parts of CopyOp shadow that come from
2643 // ConvertOp.
2644 if (CopyOp) {
2645 assert(CopyOp->getType() == I.getType())((CopyOp->getType() == I.getType()) ? static_cast<void>
(0) : __assert_fail ("CopyOp->getType() == I.getType()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2645, __PRETTY_FUNCTION__))
;
2646 assert(CopyOp->getType()->isVectorTy())((CopyOp->getType()->isVectorTy()) ? static_cast<void
> (0) : __assert_fail ("CopyOp->getType()->isVectorTy()"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2646, __PRETTY_FUNCTION__))
;
2647 Value *ResultShadow = getShadow(CopyOp);
2648 Type *EltTy = ResultShadow->getType()->getVectorElementType();
2649 for (int i = 0; i < NumUsedElements; ++i) {
2650 ResultShadow = IRB.CreateInsertElement(
2651 ResultShadow, ConstantInt::getNullValue(EltTy),
2652 ConstantInt::get(IRB.getInt32Ty(), i));
2653 }
2654 setShadow(&I, ResultShadow);
2655 setOrigin(&I, getOrigin(CopyOp));
2656 } else {
2657 setShadow(&I, getCleanShadow(&I));
2658 setOrigin(&I, getCleanOrigin());
2659 }
2660 }
2661
2662 // Given a scalar or vector, extract lower 64 bits (or less), and return all
2663 // zeroes if it is zero, and all ones otherwise.
2664 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2665 if (S->getType()->isVectorTy())
2666 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2667 assert(S->getType()->getPrimitiveSizeInBits() <= 64)((S->getType()->getPrimitiveSizeInBits() <= 64) ? static_cast
<void> (0) : __assert_fail ("S->getType()->getPrimitiveSizeInBits() <= 64"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2667, __PRETTY_FUNCTION__))
;
2668 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2669 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2670 }
2671
2672 // Given a vector, extract its first element, and return all
2673 // zeroes if it is zero, and all ones otherwise.
2674 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2675 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2676 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2677 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2678 }
2679
2680 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2681 Type *T = S->getType();
2682 assert(T->isVectorTy())((T->isVectorTy()) ? static_cast<void> (0) : __assert_fail
("T->isVectorTy()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2682, __PRETTY_FUNCTION__))
;
2683 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2684 return IRB.CreateSExt(S2, T);
2685 }
2686
2687 // Instrument vector shift instrinsic.
2688 //
2689 // This function instruments intrinsics like int_x86_avx2_psll_w.
2690 // Intrinsic shifts %In by %ShiftSize bits.
2691 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2692 // size, and the rest is ignored. Behavior is defined even if shift size is
2693 // greater than register (or field) width.
2694 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2695 assert(I.getNumArgOperands() == 2)((I.getNumArgOperands() == 2) ? static_cast<void> (0) :
__assert_fail ("I.getNumArgOperands() == 2", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2695, __PRETTY_FUNCTION__))
;
2696 IRBuilder<> IRB(&I);
2697 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2698 // Otherwise perform the same shift on S1.
2699 Value *S1 = getShadow(&I, 0);
2700 Value *S2 = getShadow(&I, 1);
2701 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2702 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2703 Value *V1 = I.getOperand(0);
2704 Value *V2 = I.getOperand(1);
2705 Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledValue(),
2706 {IRB.CreateBitCast(S1, V1->getType()), V2});
2707 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2708 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2709 setOriginForNaryOp(I);
2710 }
2711
2712 // Get an X86_MMX-sized vector type.
2713 Type *getMMXVectorTy(unsigned EltSizeInBits) {
2714 const unsigned X86_MMXSizeInBits = 64;
2715 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&((EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits
) == 0 && "Illegal MMX vector element size") ? static_cast
<void> (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2716, __PRETTY_FUNCTION__))
2716 "Illegal MMX vector element size")((EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits
) == 0 && "Illegal MMX vector element size") ? static_cast
<void> (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2716, __PRETTY_FUNCTION__))
;
2717 return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2718 X86_MMXSizeInBits / EltSizeInBits);
2719 }
2720
2721 // Returns a signed counterpart for an (un)signed-saturate-and-pack
2722 // intrinsic.
2723 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2724 switch (id) {
2725 case Intrinsic::x86_sse2_packsswb_128:
2726 case Intrinsic::x86_sse2_packuswb_128:
2727 return Intrinsic::x86_sse2_packsswb_128;
2728
2729 case Intrinsic::x86_sse2_packssdw_128:
2730 case Intrinsic::x86_sse41_packusdw:
2731 return Intrinsic::x86_sse2_packssdw_128;
2732
2733 case Intrinsic::x86_avx2_packsswb:
2734 case Intrinsic::x86_avx2_packuswb:
2735 return Intrinsic::x86_avx2_packsswb;
2736
2737 case Intrinsic::x86_avx2_packssdw:
2738 case Intrinsic::x86_avx2_packusdw:
2739 return Intrinsic::x86_avx2_packssdw;
2740
2741 case Intrinsic::x86_mmx_packsswb:
2742 case Intrinsic::x86_mmx_packuswb:
2743 return Intrinsic::x86_mmx_packsswb;
2744
2745 case Intrinsic::x86_mmx_packssdw:
2746 return Intrinsic::x86_mmx_packssdw;
2747 default:
2748 llvm_unreachable("unexpected intrinsic id")::llvm::llvm_unreachable_internal("unexpected intrinsic id", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2748)
;
2749 }
2750 }
2751
2752 // Instrument vector pack instrinsic.
2753 //
2754 // This function instruments intrinsics like x86_mmx_packsswb, that
2755 // packs elements of 2 input vectors into half as many bits with saturation.
2756 // Shadow is propagated with the signed variant of the same intrinsic applied
2757 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2758 // EltSizeInBits is used only for x86mmx arguments.
2759 void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2760 assert(I.getNumArgOperands() == 2)((I.getNumArgOperands() == 2) ? static_cast<void> (0) :
__assert_fail ("I.getNumArgOperands() == 2", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2760, __PRETTY_FUNCTION__))
;
2761 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2762 IRBuilder<> IRB(&I);
2763 Value *S1 = getShadow(&I, 0);
2764 Value *S2 = getShadow(&I, 1);
2765 assert(isX86_MMX || S1->getType()->isVectorTy())((isX86_MMX || S1->getType()->isVectorTy()) ? static_cast
<void> (0) : __assert_fail ("isX86_MMX || S1->getType()->isVectorTy()"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2765, __PRETTY_FUNCTION__))
;
2766
2767 // SExt and ICmpNE below must apply to individual elements of input vectors.
2768 // In case of x86mmx arguments, cast them to appropriate vector types and
2769 // back.
2770 Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
2771 if (isX86_MMX) {
2772 S1 = IRB.CreateBitCast(S1, T);
2773 S2 = IRB.CreateBitCast(S2, T);
2774 }
2775 Value *S1_ext = IRB.CreateSExt(
2776 IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
2777 Value *S2_ext = IRB.CreateSExt(
2778 IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
2779 if (isX86_MMX) {
2780 Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2781 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2782 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2783 }
2784
2785 Function *ShadowFn = Intrinsic::getDeclaration(
2786 F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2787
2788 Value *S =
2789 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2790 if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
2791 setShadow(&I, S);
2792 setOriginForNaryOp(I);
2793 }
2794
2795 // Instrument sum-of-absolute-differencies intrinsic.
2796 void handleVectorSadIntrinsic(IntrinsicInst &I) {
2797 const unsigned SignificantBitsPerResultElement = 16;
2798 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2799 Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
2800 unsigned ZeroBitsPerResultElement =
2801 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2802
2803 IRBuilder<> IRB(&I);
2804 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2805 S = IRB.CreateBitCast(S, ResTy);
2806 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2807 ResTy);
2808 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2809 S = IRB.CreateBitCast(S, getShadowTy(&I));
2810 setShadow(&I, S);
2811 setOriginForNaryOp(I);
2812 }
2813
2814 // Instrument multiply-add intrinsic.
2815 void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2816 unsigned EltSizeInBits = 0) {
2817 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2818 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
2819 IRBuilder<> IRB(&I);
2820 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2821 S = IRB.CreateBitCast(S, ResTy);
2822 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2823 ResTy);
2824 S = IRB.CreateBitCast(S, getShadowTy(&I));
2825 setShadow(&I, S);
2826 setOriginForNaryOp(I);
2827 }
2828
2829 // Instrument compare-packed intrinsic.
2830 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
2831 // all-ones shadow.
2832 void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
2833 IRBuilder<> IRB(&I);
2834 Type *ResTy = getShadowTy(&I);
2835 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2836 Value *S = IRB.CreateSExt(
2837 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
2838 setShadow(&I, S);
2839 setOriginForNaryOp(I);
2840 }
2841
2842 // Instrument compare-scalar intrinsic.
2843 // This handles both cmp* intrinsics which return the result in the first
2844 // element of a vector, and comi* which return the result as i32.
2845 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
2846 IRBuilder<> IRB(&I);
2847 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2848 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
2849 setShadow(&I, S);
2850 setOriginForNaryOp(I);
2851 }
2852
2853 void handleStmxcsr(IntrinsicInst &I) {
2854 IRBuilder<> IRB(&I);
2855 Value* Addr = I.getArgOperand(0);
2856 Type *Ty = IRB.getInt32Ty();
2857 Value *ShadowPtr =
2858 getShadowOriginPtr(Addr, IRB, Ty, /*Alignment*/ 1, /*isStore*/ true)
2859 .first;
2860
2861 IRB.CreateStore(getCleanShadow(Ty),
2862 IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
2863
2864 if (ClCheckAccessAddress)
2865 insertShadowCheck(Addr, &I);
2866 }
2867
2868 void handleLdmxcsr(IntrinsicInst &I) {
2869 if (!InsertChecks) return;
2870
2871 IRBuilder<> IRB(&I);
2872 Value *Addr = I.getArgOperand(0);
2873 Type *Ty = IRB.getInt32Ty();
2874 unsigned Alignment = 1;
2875 Value *ShadowPtr, *OriginPtr;
2876 std::tie(ShadowPtr, OriginPtr) =
2877 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
2878
2879 if (ClCheckAccessAddress)
2880 insertShadowCheck(Addr, &I);
2881
2882 Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
2883 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
2884 : getCleanOrigin();
2885 insertShadowCheck(Shadow, Origin, &I);
2886 }
2887
2888 void handleMaskedStore(IntrinsicInst &I) {
2889 IRBuilder<> IRB(&I);
2890 Value *V = I.getArgOperand(0);
2891 Value *Addr = I.getArgOperand(1);
2892 unsigned Align = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
2893 Value *Mask = I.getArgOperand(3);
2894 Value *Shadow = getShadow(V);
2895
2896 Value *ShadowPtr;
2897 Value *OriginPtr;
2898 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2899 Addr, IRB, Shadow->getType(), Align, /*isStore*/ true);
2900
2901 if (ClCheckAccessAddress) {
2902 insertShadowCheck(Addr, &I);
2903 // Uninitialized mask is kind of like uninitialized address, but not as
2904 // scary.
2905 insertShadowCheck(Mask, &I);
2906 }
2907
2908 IRB.CreateMaskedStore(Shadow, ShadowPtr, Align, Mask);
2909
2910 if (MS.TrackOrigins) {
2911 auto &DL = F.getParent()->getDataLayout();
2912 paintOrigin(IRB, getOrigin(V), OriginPtr,
2913 DL.getTypeStoreSize(Shadow->getType()),
2914 std::max(Align, kMinOriginAlignment));
2915 }
2916 }
2917
2918 bool handleMaskedLoad(IntrinsicInst &I) {
2919 IRBuilder<> IRB(&I);
2920 Value *Addr = I.getArgOperand(0);
2921 unsigned Align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
2922 Value *Mask = I.getArgOperand(2);
2923 Value *PassThru = I.getArgOperand(3);
2924
2925 Type *ShadowTy = getShadowTy(&I);
2926 Value *ShadowPtr, *OriginPtr;
2927 if (PropagateShadow) {
2928 std::tie(ShadowPtr, OriginPtr) =
2929 getShadowOriginPtr(Addr, IRB, ShadowTy, Align, /*isStore*/ false);
2930 setShadow(&I, IRB.CreateMaskedLoad(ShadowPtr, Align, Mask,
2931 getShadow(PassThru), "_msmaskedld"));
2932 } else {
2933 setShadow(&I, getCleanShadow(&I));
2934 }
2935
2936 if (ClCheckAccessAddress) {
2937 insertShadowCheck(Addr, &I);
2938 insertShadowCheck(Mask, &I);
2939 }
2940
2941 if (MS.TrackOrigins) {
2942 if (PropagateShadow) {
2943 // Choose between PassThru's and the loaded value's origins.
2944 Value *MaskedPassThruShadow = IRB.CreateAnd(
2945 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
2946
2947 Value *Acc = IRB.CreateExtractElement(
2948 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2949 for (int i = 1, N = PassThru->getType()->getVectorNumElements(); i < N;
2950 ++i) {
2951 Value *More = IRB.CreateExtractElement(
2952 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2953 Acc = IRB.CreateOr(Acc, More);
2954 }
2955
2956 Value *Origin = IRB.CreateSelect(
2957 IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
2958 getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr));
2959
2960 setOrigin(&I, Origin);
2961 } else {
2962 setOrigin(&I, getCleanOrigin());
2963 }
2964 }
2965 return true;
2966 }
2967
2968 // Instrument BMI / BMI2 intrinsics.
2969 // All of these intrinsics are Z = I(X, Y)
2970 // where the types of all operands and the result match, and are either i32 or i64.
2971 // The following instrumentation happens to work for all of them:
2972 // Sz = I(Sx, Y) | (sext (Sy != 0))
2973 void handleBmiIntrinsic(IntrinsicInst &I) {
2974 IRBuilder<> IRB(&I);
2975 Type *ShadowTy = getShadowTy(&I);
2976
2977 // If any bit of the mask operand is poisoned, then the whole thing is.
2978 Value *SMask = getShadow(&I, 1);
2979 SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
2980 ShadowTy);
2981 // Apply the same intrinsic to the shadow of the first operand.
2982 Value *S = IRB.CreateCall(I.getCalledFunction(),
2983 {getShadow(&I, 0), I.getOperand(1)});
2984 S = IRB.CreateOr(SMask, S);
2985 setShadow(&I, S);
2986 setOriginForNaryOp(I);
2987 }
2988
2989 void visitIntrinsicInst(IntrinsicInst &I) {
2990 switch (I.getIntrinsicID()) {
2991 case Intrinsic::lifetime_start:
2992 handleLifetimeStart(I);
2993 break;
2994 case Intrinsic::bswap:
2995 handleBswap(I);
2996 break;
2997 case Intrinsic::masked_store:
2998 handleMaskedStore(I);
2999 break;
3000 case Intrinsic::masked_load:
3001 handleMaskedLoad(I);
3002 break;
3003 case Intrinsic::x86_sse_stmxcsr:
3004 handleStmxcsr(I);
3005 break;
3006 case Intrinsic::x86_sse_ldmxcsr:
3007 handleLdmxcsr(I);
3008 break;
3009 case Intrinsic::x86_avx512_vcvtsd2usi64:
3010 case Intrinsic::x86_avx512_vcvtsd2usi32:
3011 case Intrinsic::x86_avx512_vcvtss2usi64:
3012 case Intrinsic::x86_avx512_vcvtss2usi32:
3013 case Intrinsic::x86_avx512_cvttss2usi64:
3014 case Intrinsic::x86_avx512_cvttss2usi:
3015 case Intrinsic::x86_avx512_cvttsd2usi64:
3016 case Intrinsic::x86_avx512_cvttsd2usi:
3017 case Intrinsic::x86_avx512_cvtusi2ss:
3018 case Intrinsic::x86_avx512_cvtusi642sd:
3019 case Intrinsic::x86_avx512_cvtusi642ss:
3020 case Intrinsic::x86_sse2_cvtsd2si64:
3021 case Intrinsic::x86_sse2_cvtsd2si:
3022 case Intrinsic::x86_sse2_cvtsd2ss:
3023 case Intrinsic::x86_sse2_cvttsd2si64:
3024 case Intrinsic::x86_sse2_cvttsd2si:
3025 case Intrinsic::x86_sse_cvtss2si64:
3026 case Intrinsic::x86_sse_cvtss2si:
3027 case Intrinsic::x86_sse_cvttss2si64:
3028 case Intrinsic::x86_sse_cvttss2si:
3029 handleVectorConvertIntrinsic(I, 1);
3030 break;
3031 case Intrinsic::x86_sse_cvtps2pi:
3032 case Intrinsic::x86_sse_cvttps2pi:
3033 handleVectorConvertIntrinsic(I, 2);
3034 break;
3035
3036 case Intrinsic::x86_avx512_psll_w_512:
3037 case Intrinsic::x86_avx512_psll_d_512:
3038 case Intrinsic::x86_avx512_psll_q_512:
3039 case Intrinsic::x86_avx512_pslli_w_512:
3040 case Intrinsic::x86_avx512_pslli_d_512:
3041 case Intrinsic::x86_avx512_pslli_q_512:
3042 case Intrinsic::x86_avx512_psrl_w_512:
3043 case Intrinsic::x86_avx512_psrl_d_512:
3044 case Intrinsic::x86_avx512_psrl_q_512:
3045 case Intrinsic::x86_avx512_psra_w_512:
3046 case Intrinsic::x86_avx512_psra_d_512:
3047 case Intrinsic::x86_avx512_psra_q_512:
3048 case Intrinsic::x86_avx512_psrli_w_512:
3049 case Intrinsic::x86_avx512_psrli_d_512:
3050 case Intrinsic::x86_avx512_psrli_q_512:
3051 case Intrinsic::x86_avx512_psrai_w_512:
3052 case Intrinsic::x86_avx512_psrai_d_512:
3053 case Intrinsic::x86_avx512_psrai_q_512:
3054 case Intrinsic::x86_avx512_psra_q_256:
3055 case Intrinsic::x86_avx512_psra_q_128:
3056 case Intrinsic::x86_avx512_psrai_q_256:
3057 case Intrinsic::x86_avx512_psrai_q_128:
3058 case Intrinsic::x86_avx2_psll_w:
3059 case Intrinsic::x86_avx2_psll_d:
3060 case Intrinsic::x86_avx2_psll_q:
3061 case Intrinsic::x86_avx2_pslli_w:
3062 case Intrinsic::x86_avx2_pslli_d:
3063 case Intrinsic::x86_avx2_pslli_q:
3064 case Intrinsic::x86_avx2_psrl_w:
3065 case Intrinsic::x86_avx2_psrl_d:
3066 case Intrinsic::x86_avx2_psrl_q:
3067 case Intrinsic::x86_avx2_psra_w:
3068 case Intrinsic::x86_avx2_psra_d:
3069 case Intrinsic::x86_avx2_psrli_w:
3070 case Intrinsic::x86_avx2_psrli_d:
3071 case Intrinsic::x86_avx2_psrli_q:
3072 case Intrinsic::x86_avx2_psrai_w:
3073 case Intrinsic::x86_avx2_psrai_d:
3074 case Intrinsic::x86_sse2_psll_w:
3075 case Intrinsic::x86_sse2_psll_d:
3076 case Intrinsic::x86_sse2_psll_q:
3077 case Intrinsic::x86_sse2_pslli_w:
3078 case Intrinsic::x86_sse2_pslli_d:
3079 case Intrinsic::x86_sse2_pslli_q:
3080 case Intrinsic::x86_sse2_psrl_w:
3081 case Intrinsic::x86_sse2_psrl_d:
3082 case Intrinsic::x86_sse2_psrl_q:
3083 case Intrinsic::x86_sse2_psra_w:
3084 case Intrinsic::x86_sse2_psra_d:
3085 case Intrinsic::x86_sse2_psrli_w:
3086 case Intrinsic::x86_sse2_psrli_d:
3087 case Intrinsic::x86_sse2_psrli_q:
3088 case Intrinsic::x86_sse2_psrai_w:
3089 case Intrinsic::x86_sse2_psrai_d:
3090 case Intrinsic::x86_mmx_psll_w:
3091 case Intrinsic::x86_mmx_psll_d:
3092 case Intrinsic::x86_mmx_psll_q:
3093 case Intrinsic::x86_mmx_pslli_w:
3094 case Intrinsic::x86_mmx_pslli_d:
3095 case Intrinsic::x86_mmx_pslli_q:
3096 case Intrinsic::x86_mmx_psrl_w:
3097 case Intrinsic::x86_mmx_psrl_d:
3098 case Intrinsic::x86_mmx_psrl_q:
3099 case Intrinsic::x86_mmx_psra_w:
3100 case Intrinsic::x86_mmx_psra_d:
3101 case Intrinsic::x86_mmx_psrli_w:
3102 case Intrinsic::x86_mmx_psrli_d:
3103 case Intrinsic::x86_mmx_psrli_q:
3104 case Intrinsic::x86_mmx_psrai_w:
3105 case Intrinsic::x86_mmx_psrai_d:
3106 handleVectorShiftIntrinsic(I, /* Variable */ false);
3107 break;
3108 case Intrinsic::x86_avx2_psllv_d:
3109 case Intrinsic::x86_avx2_psllv_d_256:
3110 case Intrinsic::x86_avx512_psllv_d_512:
3111 case Intrinsic::x86_avx2_psllv_q:
3112 case Intrinsic::x86_avx2_psllv_q_256:
3113 case Intrinsic::x86_avx512_psllv_q_512:
3114 case Intrinsic::x86_avx2_psrlv_d:
3115 case Intrinsic::x86_avx2_psrlv_d_256:
3116 case Intrinsic::x86_avx512_psrlv_d_512:
3117 case Intrinsic::x86_avx2_psrlv_q:
3118 case Intrinsic::x86_avx2_psrlv_q_256:
3119 case Intrinsic::x86_avx512_psrlv_q_512:
3120 case Intrinsic::x86_avx2_psrav_d:
3121 case Intrinsic::x86_avx2_psrav_d_256:
3122 case Intrinsic::x86_avx512_psrav_d_512:
3123 case Intrinsic::x86_avx512_psrav_q_128:
3124 case Intrinsic::x86_avx512_psrav_q_256:
3125 case Intrinsic::x86_avx512_psrav_q_512:
3126 handleVectorShiftIntrinsic(I, /* Variable */ true);
3127 break;
3128
3129 case Intrinsic::x86_sse2_packsswb_128:
3130 case Intrinsic::x86_sse2_packssdw_128:
3131 case Intrinsic::x86_sse2_packuswb_128:
3132 case Intrinsic::x86_sse41_packusdw:
3133 case Intrinsic::x86_avx2_packsswb:
3134 case Intrinsic::x86_avx2_packssdw:
3135 case Intrinsic::x86_avx2_packuswb:
3136 case Intrinsic::x86_avx2_packusdw:
3137 handleVectorPackIntrinsic(I);
3138 break;
3139
3140 case Intrinsic::x86_mmx_packsswb:
3141 case Intrinsic::x86_mmx_packuswb:
3142 handleVectorPackIntrinsic(I, 16);
3143 break;
3144
3145 case Intrinsic::x86_mmx_packssdw:
3146 handleVectorPackIntrinsic(I, 32);
3147 break;
3148
3149 case Intrinsic::x86_mmx_psad_bw:
3150 case Intrinsic::x86_sse2_psad_bw:
3151 case Intrinsic::x86_avx2_psad_bw:
3152 handleVectorSadIntrinsic(I);
3153 break;
3154
3155 case Intrinsic::x86_sse2_pmadd_wd:
3156 case Intrinsic::x86_avx2_pmadd_wd:
3157 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3158 case Intrinsic::x86_avx2_pmadd_ub_sw:
3159 handleVectorPmaddIntrinsic(I);
3160 break;
3161
3162 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3163 handleVectorPmaddIntrinsic(I, 8);
3164 break;
3165
3166 case Intrinsic::x86_mmx_pmadd_wd:
3167 handleVectorPmaddIntrinsic(I, 16);
3168 break;
3169
3170 case Intrinsic::x86_sse_cmp_ss:
3171 case Intrinsic::x86_sse2_cmp_sd:
3172 case Intrinsic::x86_sse_comieq_ss:
3173 case Intrinsic::x86_sse_comilt_ss:
3174 case Intrinsic::x86_sse_comile_ss:
3175 case Intrinsic::x86_sse_comigt_ss:
3176 case Intrinsic::x86_sse_comige_ss:
3177 case Intrinsic::x86_sse_comineq_ss:
3178 case Intrinsic::x86_sse_ucomieq_ss:
3179 case Intrinsic::x86_sse_ucomilt_ss:
3180 case Intrinsic::x86_sse_ucomile_ss:
3181 case Intrinsic::x86_sse_ucomigt_ss:
3182 case Intrinsic::x86_sse_ucomige_ss:
3183 case Intrinsic::x86_sse_ucomineq_ss:
3184 case Intrinsic::x86_sse2_comieq_sd:
3185 case Intrinsic::x86_sse2_comilt_sd:
3186 case Intrinsic::x86_sse2_comile_sd:
3187 case Intrinsic::x86_sse2_comigt_sd:
3188 case Intrinsic::x86_sse2_comige_sd:
3189 case Intrinsic::x86_sse2_comineq_sd:
3190 case Intrinsic::x86_sse2_ucomieq_sd:
3191 case Intrinsic::x86_sse2_ucomilt_sd:
3192 case Intrinsic::x86_sse2_ucomile_sd:
3193 case Intrinsic::x86_sse2_ucomigt_sd:
3194 case Intrinsic::x86_sse2_ucomige_sd:
3195 case Intrinsic::x86_sse2_ucomineq_sd:
3196 handleVectorCompareScalarIntrinsic(I);
3197 break;
3198
3199 case Intrinsic::x86_sse_cmp_ps:
3200 case Intrinsic::x86_sse2_cmp_pd:
3201 // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
3202 // generates reasonably looking IR that fails in the backend with "Do not
3203 // know how to split the result of this operator!".
3204 handleVectorComparePackedIntrinsic(I);
3205 break;
3206
3207 case Intrinsic::x86_bmi_bextr_32:
3208 case Intrinsic::x86_bmi_bextr_64:
3209 case Intrinsic::x86_bmi_bzhi_32:
3210 case Intrinsic::x86_bmi_bzhi_64:
3211 case Intrinsic::x86_bmi_pdep_32:
3212 case Intrinsic::x86_bmi_pdep_64:
3213 case Intrinsic::x86_bmi_pext_32:
3214 case Intrinsic::x86_bmi_pext_64:
3215 handleBmiIntrinsic(I);
3216 break;
3217
3218 case Intrinsic::is_constant:
3219 // The result of llvm.is.constant() is always defined.
3220 setShadow(&I, getCleanShadow(&I));
3221 setOrigin(&I, getCleanOrigin());
3222 break;
3223
3224 default:
3225 if (!handleUnknownIntrinsic(I))
3226 visitInstruction(I);
3227 break;
3228 }
3229 }
3230
3231 void visitCallSite(CallSite CS) {
3232 Instruction &I = *CS.getInstruction();
3233 assert(!I.getMetadata("nosanitize"))((!I.getMetadata("nosanitize")) ? static_cast<void> (0)
: __assert_fail ("!I.getMetadata(\"nosanitize\")", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3233, __PRETTY_FUNCTION__))
;
3234 assert((CS.isCall() || CS.isInvoke()) && "Unknown type of CallSite")(((CS.isCall() || CS.isInvoke()) && "Unknown type of CallSite"
) ? static_cast<void> (0) : __assert_fail ("(CS.isCall() || CS.isInvoke()) && \"Unknown type of CallSite\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3234, __PRETTY_FUNCTION__))
;
3235 if (CS.isCall()) {
3236 CallInst *Call = cast<CallInst>(&I);
3237
3238 // For inline asm, do the usual thing: check argument shadow and mark all
3239 // outputs as clean. Note that any side effects of the inline asm that are
3240 // not immediately visible in its constraints are not handled.
3241 if (Call->isInlineAsm()) {
3242 if (ClHandleAsmConservative && MS.CompileKernel)
3243 visitAsmInstruction(I);
3244 else
3245 visitInstruction(I);
3246 return;
3247 }
3248
3249 assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere")((!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("!isa<IntrinsicInst>(&I) && \"intrinsics are handled elsewhere\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3249, __PRETTY_FUNCTION__))
;
3250
3251 // We are going to insert code that relies on the fact that the callee
3252 // will become a non-readonly function after it is instrumented by us. To
3253 // prevent this code from being optimized out, mark that function
3254 // non-readonly in advance.
3255 if (Function *Func = Call->getCalledFunction()) {
3256 // Clear out readonly/readnone attributes.
3257 AttrBuilder B;
3258 B.addAttribute(Attribute::ReadOnly)
3259 .addAttribute(Attribute::ReadNone);
3260 Func->removeAttributes(AttributeList::FunctionIndex, B);
3261 }
3262
3263 maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
3264 }
3265 IRBuilder<> IRB(&I);
3266
3267 unsigned ArgOffset = 0;
3268 LLVM_DEBUG(dbgs() << " CallSite: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " CallSite: " << I <<
"\n"; } } while (false)
;
3269 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3270 ArgIt != End; ++ArgIt) {
3271 Value *A = *ArgIt;
3272 unsigned i = ArgIt - CS.arg_begin();
3273 if (!A->getType()->isSized()) {
3274 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Arg " << i << " is not sized: "
<< I << "\n"; } } while (false)
;
3275 continue;
3276 }
3277 unsigned Size = 0;
3278 Value *Store = nullptr;
3279 // Compute the Shadow for arg even if it is ByVal, because
3280 // in that case getShadow() will copy the actual arg shadow to
3281 // __msan_param_tls.
3282 Value *ArgShadow = getShadow(A);
3283 Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
3284 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *Ado { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Arg#" << i << ": "
<< *A << " Shadow: " << *ArgShadow <<
"\n"; } } while (false)
3285 << " Shadow: " << *ArgShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Arg#" << i << ": "
<< *A << " Shadow: " << *ArgShadow <<
"\n"; } } while (false)
;
3286 bool ArgIsInitialized = false;
3287 const DataLayout &DL = F.getParent()->getDataLayout();
3288 if (CS.paramHasAttr(i, Attribute::ByVal)) {
3289 assert(A->getType()->isPointerTy() &&((A->getType()->isPointerTy() && "ByVal argument is not a pointer!"
) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3290, __PRETTY_FUNCTION__))
3290 "ByVal argument is not a pointer!")((A->getType()->isPointerTy() && "ByVal argument is not a pointer!"
) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3290, __PRETTY_FUNCTION__))
;
3291 Size = DL.getTypeAllocSize(A->getType()->getPointerElementType());
3292 if (ArgOffset + Size > kParamTLSSize) break;
3293 unsigned ParamAlignment = CS.getParamAlignment(i);
3294 unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment);
3295 Value *AShadowPtr =
3296 getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
3297 /*isStore*/ false)
3298 .first;
3299
3300 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
3301 Alignment, Size);
3302 // TODO(glider): need to copy origins.
3303 } else {
3304 Size = DL.getTypeAllocSize(A->getType());
3305 if (ArgOffset + Size > kParamTLSSize) break;
3306 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
3307 kShadowTLSAlignment);
3308 Constant *Cst = dyn_cast<Constant>(ArgShadow);
3309 if (Cst && Cst->isNullValue()) ArgIsInitialized = true;
3310 }
3311 if (MS.TrackOrigins && !ArgIsInitialized)
3312 IRB.CreateStore(getOrigin(A),
3313 getOriginPtrForArgument(A, IRB, ArgOffset));
3314 (void)Store;
3315 assert(Size != 0 && Store != nullptr)((Size != 0 && Store != nullptr) ? static_cast<void
> (0) : __assert_fail ("Size != 0 && Store != nullptr"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3315, __PRETTY_FUNCTION__))
;
3316 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Param:" << *Store <<
"\n"; } } while (false)
;
3317 ArgOffset += alignTo(Size, 8);
3318 }
3319 LLVM_DEBUG(dbgs() << " done with call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " done with call args\n"; } } while
(false)
;
3320
3321 FunctionType *FT = CS.getFunctionType();
3322 if (FT->isVarArg()) {
3323 VAHelper->visitCallSite(CS, IRB);
3324 }
3325
3326 // Now, get the shadow for the RetVal.
3327 if (!I.getType()->isSized()) return;
3328 // Don't emit the epilogue for musttail call returns.
3329 if (CS.isCall() && cast<CallInst>(&I)->isMustTailCall()) return;
3330 IRBuilder<> IRBBefore(&I);
3331 // Until we have full dynamic coverage, make sure the retval shadow is 0.
3332 Value *Base = getShadowPtrForRetval(&I, IRBBefore);
3333 IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, kShadowTLSAlignment);
3334 BasicBlock::iterator NextInsn;
3335 if (CS.isCall()) {
3336 NextInsn = ++I.getIterator();
3337 assert(NextInsn != I.getParent()->end())((NextInsn != I.getParent()->end()) ? static_cast<void>
(0) : __assert_fail ("NextInsn != I.getParent()->end()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3337, __PRETTY_FUNCTION__))
;
3338 } else {
3339 BasicBlock *NormalDest = cast<InvokeInst>(&I)->getNormalDest();
3340 if (!NormalDest->getSinglePredecessor()) {
3341 // FIXME: this case is tricky, so we are just conservative here.
3342 // Perhaps we need to split the edge between this BB and NormalDest,
3343 // but a naive attempt to use SplitEdge leads to a crash.
3344 setShadow(&I, getCleanShadow(&I));
3345 setOrigin(&I, getCleanOrigin());
3346 return;
3347 }
3348 // FIXME: NextInsn is likely in a basic block that has not been visited yet.
3349 // Anything inserted there will be instrumented by MSan later!
3350 NextInsn = NormalDest->getFirstInsertionPt();
3351 assert(NextInsn != NormalDest->end() &&((NextInsn != NormalDest->end() && "Could not find insertion point for retval shadow load"
) ? static_cast<void> (0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3352, __PRETTY_FUNCTION__))
3352 "Could not find insertion point for retval shadow load")((NextInsn != NormalDest->end() && "Could not find insertion point for retval shadow load"
) ? static_cast<void> (0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3352, __PRETTY_FUNCTION__))
;
3353 }
3354 IRBuilder<> IRBAfter(&*NextInsn);
3355 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
3356 getShadowTy(&I), getShadowPtrForRetval(&I, IRBAfter),
3357 kShadowTLSAlignment, "_msret");
3358 setShadow(&I, RetvalShadow);
3359 if (MS.TrackOrigins)
3360 setOrigin(&I, IRBAfter.CreateLoad(MS.OriginTy,
3361 getOriginPtrForRetval(IRBAfter)));
3362 }
3363
3364 bool isAMustTailRetVal(Value *RetVal) {
3365 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
3366 RetVal = I->getOperand(0);
3367 }
3368 if (auto *I = dyn_cast<CallInst>(RetVal)) {
3369 return I->isMustTailCall();
3370 }
3371 return false;
3372 }
3373
3374 void visitReturnInst(ReturnInst &I) {
3375 IRBuilder<> IRB(&I);
3376 Value *RetVal = I.getReturnValue();
3377 if (!RetVal) return;
3378 // Don't emit the epilogue for musttail call returns.
3379 if (isAMustTailRetVal(RetVal)) return;
3380 Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
3381 if (CheckReturnValue) {
3382 insertShadowCheck(RetVal, &I);
3383 Value *Shadow = getCleanShadow(RetVal);
3384 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3385 } else {
3386 Value *Shadow = getShadow(RetVal);
3387 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3388 if (MS.TrackOrigins)
3389 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
3390 }
3391 }
3392
3393 void visitPHINode(PHINode &I) {
3394 IRBuilder<> IRB(&I);
3395 if (!PropagateShadow) {
3396 setShadow(&I, getCleanShadow(&I));
3397 setOrigin(&I, getCleanOrigin());
3398 return;
3399 }
3400
3401 ShadowPHINodes.push_back(&I);
3402 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
3403 "_msphi_s"));
3404 if (MS.TrackOrigins)
3405 setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
3406 "_msphi_o"));
3407 }
3408
3409 Value *getLocalVarDescription(AllocaInst &I) {
3410 SmallString<2048> StackDescriptionStorage;
3411 raw_svector_ostream StackDescription(StackDescriptionStorage);
3412 // We create a string with a description of the stack allocation and
3413 // pass it into __msan_set_alloca_origin.
3414 // It will be printed by the run-time if stack-originated UMR is found.
3415 // The first 4 bytes of the string are set to '----' and will be replaced
3416 // by __msan_va_arg_overflow_size_tls at the first call.
3417 StackDescription << "----" << I.getName() << "@" << F.getName();
3418 return createPrivateNonConstGlobalForString(*F.getParent(),
3419 StackDescription.str());
3420 }
3421
3422 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3423 if (PoisonStack && ClPoisonStackWithCall) {
3424 IRB.CreateCall(MS.MsanPoisonStackFn,
3425 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3426 } else {
3427 Value *ShadowBase, *OriginBase;
3428 std::tie(ShadowBase, OriginBase) =
3429 getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(), 1, /*isStore*/ true);
3430
3431 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
3432 IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlignment());
3433 }
3434
3435 if (PoisonStack && MS.TrackOrigins) {
3436 Value *Descr = getLocalVarDescription(I);
3437 IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
3438 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3439 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
3440 IRB.CreatePointerCast(&F, MS.IntptrTy)});
3441 }
3442 }
3443
3444 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3445 Value *Descr = getLocalVarDescription(I);
3446 if (PoisonStack) {
3447 IRB.CreateCall(MS.MsanPoisonAllocaFn,
3448 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3449 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
3450 } else {
3451 IRB.CreateCall(MS.MsanUnpoisonAllocaFn,
3452 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3453 }
3454 }
3455
3456 void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
3457 if (!InsPoint)
3458 InsPoint = &I;
3459 IRBuilder<> IRB(InsPoint->getNextNode());
3460 const DataLayout &DL = F.getParent()->getDataLayout();
3461 uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
3462 Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
3463 if (I.isArrayAllocation())
3464 Len = IRB.CreateMul(Len, I.getArraySize());
3465
3466 if (MS.CompileKernel)
3467 poisonAllocaKmsan(I, IRB, Len);
3468 else
3469 poisonAllocaUserspace(I, IRB, Len);
3470 }
3471
3472 void visitAllocaInst(AllocaInst &I) {
3473 setShadow(&I, getCleanShadow(&I));
3474 setOrigin(&I, getCleanOrigin());
3475 // We'll get to this alloca later unless it's poisoned at the corresponding
3476 // llvm.lifetime.start.
3477 AllocaSet.insert(&I);
3478 }
3479
3480 void visitSelectInst(SelectInst& I) {
3481 IRBuilder<> IRB(&I);
3482 // a = select b, c, d
3483 Value *B = I.getCondition();
3484 Value *C = I.getTrueValue();
3485 Value *D = I.getFalseValue();
3486 Value *Sb = getShadow(B);
3487 Value *Sc = getShadow(C);
3488 Value *Sd = getShadow(D);
3489
3490 // Result shadow if condition shadow is 0.
3491 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
3492 Value *Sa1;
3493 if (I.getType()->isAggregateType()) {
3494 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
3495 // an extra "select". This results in much more compact IR.
3496 // Sa = select Sb, poisoned, (select b, Sc, Sd)
3497 Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
3498 } else {
3499 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
3500 // If Sb (condition is poisoned), look for bits in c and d that are equal
3501 // and both unpoisoned.
3502 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
3503
3504 // Cast arguments to shadow-compatible type.
3505 C = CreateAppToShadowCast(IRB, C);
3506 D = CreateAppToShadowCast(IRB, D);
3507
3508 // Result shadow if condition shadow is 1.
3509 Sa1 = IRB.CreateOr(IRB.CreateXor(C, D), IRB.CreateOr(Sc, Sd));
3510 }
3511 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
3512 setShadow(&I, Sa);
3513 if (MS.TrackOrigins) {
3514 // Origins are always i32, so any vector conditions must be flattened.
3515 // FIXME: consider tracking vector origins for app vectors?
3516 if (B->getType()->isVectorTy()) {
3517 Type *FlatTy = getShadowTyNoVec(B->getType());
3518 B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
3519 ConstantInt::getNullValue(FlatTy));
3520 Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
3521 ConstantInt::getNullValue(FlatTy));
3522 }
3523 // a = select b, c, d
3524 // Oa = Sb ? Ob : (b ? Oc : Od)
3525 setOrigin(
3526 &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
3527 IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
3528 getOrigin(I.getFalseValue()))));
3529 }
3530 }
3531
3532 void visitLandingPadInst(LandingPadInst &I) {
3533 // Do nothing.
3534 // See https://github.com/google/sanitizers/issues/504
3535 setShadow(&I, getCleanShadow(&I));
3536 setOrigin(&I, getCleanOrigin());
3537 }
3538
3539 void visitCatchSwitchInst(CatchSwitchInst &I) {
3540 setShadow(&I, getCleanShadow(&I));
3541 setOrigin(&I, getCleanOrigin());
3542 }
3543
3544 void visitFuncletPadInst(FuncletPadInst &I) {
3545 setShadow(&I, getCleanShadow(&I));
3546 setOrigin(&I, getCleanOrigin());
3547 }
3548
3549 void visitGetElementPtrInst(GetElementPtrInst &I) {
3550 handleShadowOr(I);
3551 }
3552
3553 void visitExtractValueInst(ExtractValueInst &I) {
3554 IRBuilder<> IRB(&I);
3555 Value *Agg = I.getAggregateOperand();
3556 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "ExtractValue: " << I <<
"\n"; } } while (false)
;
3557 Value *AggShadow = getShadow(Agg);
3558 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " AggShadow: " << *AggShadow
<< "\n"; } } while (false)
;
3559 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
3560 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ResShadow: " << *ResShadow
<< "\n"; } } while (false)
;
3561 setShadow(&I, ResShadow);
3562 setOriginForNaryOp(I);
3563 }
3564
3565 void visitInsertValueInst(InsertValueInst &I) {
3566 IRBuilder<> IRB(&I);
3567 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "InsertValue: " << I <<
"\n"; } } while (false)
;
3568 Value *AggShadow = getShadow(I.getAggregateOperand());
3569 Value *InsShadow = getShadow(I.getInsertedValueOperand());
3570 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " AggShadow: " << *AggShadow
<< "\n"; } } while (false)
;
3571 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " InsShadow: " << *InsShadow
<< "\n"; } } while (false)
;
3572 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
3573 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Res: " << *Res <<
"\n"; } } while (false)
;
3574 setShadow(&I, Res);
3575 setOriginForNaryOp(I);
3576 }
3577
3578 void dumpInst(Instruction &I) {
3579 if (CallInst *CI = dyn_cast<CallInst>(&I)) {
3580 errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
3581 } else {
3582 errs() << "ZZZ " << I.getOpcodeName() << "\n";
3583 }
3584 errs() << "QQQ " << I << "\n";
3585 }
3586
3587 void visitResumeInst(ResumeInst &I) {
3588 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Resume: " << I << "\n"
; } } while (false)
;
3589 // Nothing to do here.
3590 }
3591
3592 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
3593 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "CleanupReturn: " << CRI <<
"\n"; } } while (false)
;
3594 // Nothing to do here.
3595 }
3596
3597 void visitCatchReturnInst(CatchReturnInst &CRI) {
3598 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "CatchReturn: " << CRI <<
"\n"; } } while (false)
;
3599 // Nothing to do here.
3600 }
3601
3602 void instrumentAsmArgument(Value *Operand, Instruction &I, IRBuilder<> &IRB,
3603 const DataLayout &DL, bool isOutput) {
3604 // For each assembly argument, we check its value for being initialized.
3605 // If the argument is a pointer, we assume it points to a single element
3606 // of the corresponding type (or to a 8-byte word, if the type is unsized).
3607 // Each such pointer is instrumented with a call to the runtime library.
3608 Type *OpType = Operand->getType();
3609 // Check the operand value itself.
3610 insertShadowCheck(Operand, &I);
3611 if (!OpType->isPointerTy() || !isOutput) {
3612 assert(!isOutput)((!isOutput) ? static_cast<void> (0) : __assert_fail ("!isOutput"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3612, __PRETTY_FUNCTION__))
;
3613 return;
3614 }
3615 Type *ElType = OpType->getPointerElementType();
3616 if (!ElType->isSized())
3617 return;
3618 int Size = DL.getTypeStoreSize(ElType);
3619 Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy());
3620 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
3621 IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
3622 }
3623
3624 /// Get the number of output arguments returned by pointers.
3625 int getNumOutputArgs(InlineAsm *IA, CallInst *CI) {
3626 int NumRetOutputs = 0;
3627 int NumOutputs = 0;
3628 Type *RetTy = dyn_cast<Value>(CI)->getType();
3629 if (!RetTy->isVoidTy()) {
3630 // Register outputs are returned via the CallInst return value.
3631 StructType *ST = dyn_cast_or_null<StructType>(RetTy);
3632 if (ST)
3633 NumRetOutputs = ST->getNumElements();
3634 else
3635 NumRetOutputs = 1;
3636 }
3637 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
3638 for (size_t i = 0, n = Constraints.size(); i < n; i++) {
3639 InlineAsm::ConstraintInfo Info = Constraints[i];
3640 switch (Info.Type) {
3641 case InlineAsm::isOutput:
3642 NumOutputs++;
3643 break;
3644 default:
3645 break;
3646 }
3647 }
3648 return NumOutputs - NumRetOutputs;
3649 }
3650
3651 void visitAsmInstruction(Instruction &I) {
3652 // Conservative inline assembly handling: check for poisoned shadow of
3653 // asm() arguments, then unpoison the result and all the memory locations
3654 // pointed to by those arguments.
3655 // An inline asm() statement in C++ contains lists of input and output
3656 // arguments used by the assembly code. These are mapped to operands of the
3657 // CallInst as follows:
3658 // - nR register outputs ("=r) are returned by value in a single structure
3659 // (SSA value of the CallInst);
3660 // - nO other outputs ("=m" and others) are returned by pointer as first
3661 // nO operands of the CallInst;
3662 // - nI inputs ("r", "m" and others) are passed to CallInst as the
3663 // remaining nI operands.
3664 // The total number of asm() arguments in the source is nR+nO+nI, and the
3665 // corresponding CallInst has nO+nI+1 operands (the last operand is the
3666 // function to be called).
3667 const DataLayout &DL = F.getParent()->getDataLayout();
3668 CallInst *CI = dyn_cast<CallInst>(&I);
3669 IRBuilder<> IRB(&I);
3670 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue());
3671 int OutputArgs = getNumOutputArgs(IA, CI);
3672 // The last operand of a CallInst is the function itself.
3673 int NumOperands = CI->getNumOperands() - 1;
3674
3675 // Check input arguments. Doing so before unpoisoning output arguments, so
3676 // that we won't overwrite uninit values before checking them.
3677 for (int i = OutputArgs; i < NumOperands; i++) {
3678 Value *Operand = CI->getOperand(i);
3679 instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ false);
3680 }
3681 // Unpoison output arguments. This must happen before the actual InlineAsm
3682 // call, so that the shadow for memory published in the asm() statement
3683 // remains valid.
3684 for (int i = 0; i < OutputArgs; i++) {
3685 Value *Operand = CI->getOperand(i);
3686 instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ true);
3687 }
3688
3689 setShadow(&I, getCleanShadow(&I));
3690 setOrigin(&I, getCleanOrigin());
3691 }
3692
3693 void visitInstruction(Instruction &I) {
3694 // Everything else: stop propagating and check for poisoned shadow.
3695 if (ClDumpStrictInstructions)
3696 dumpInst(I);
3697 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "DEFAULT: " << I << "\n"
; } } while (false)
;
3698 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
3699 Value *Operand = I.getOperand(i);
3700 if (Operand->getType()->isSized())
3701 insertShadowCheck(Operand, &I);
3702 }
3703 setShadow(&I, getCleanShadow(&I));
3704 setOrigin(&I, getCleanOrigin());
3705 }
3706};
3707
3708/// AMD64-specific implementation of VarArgHelper.
3709struct VarArgAMD64Helper : public VarArgHelper {
3710 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
3711 // See a comment in visitCallSite for more details.
3712 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
3713 static const unsigned AMD64FpEndOffsetSSE = 176;
3714 // If SSE is disabled, fp_offset in va_list is zero.
3715 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
3716
3717 unsigned AMD64FpEndOffset;
3718 Function &F;
3719 MemorySanitizer &MS;
3720 MemorySanitizerVisitor &MSV;
3721 Value *VAArgTLSCopy = nullptr;
3722 Value *VAArgTLSOriginCopy = nullptr;
3723 Value *VAArgOverflowSize = nullptr;
3724
3725 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3726
3727 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
3728
3729 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
3730 MemorySanitizerVisitor &MSV)
3731 : F(F), MS(MS), MSV(MSV) {
3732 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
3733 for (const auto &Attr : F.getAttributes().getFnAttributes()) {
3734 if (Attr.isStringAttribute() &&
3735 (Attr.getKindAsString() == "target-features")) {
3736 if (Attr.getValueAsString().contains("-sse"))
3737 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
3738 break;
3739 }
3740 }
3741 }
3742
3743 ArgKind classifyArgument(Value* arg) {
3744 // A very rough approximation of X86_64 argument classification rules.
3745 Type *T = arg->getType();
3746 if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
3747 return AK_FloatingPoint;
3748 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
3749 return AK_GeneralPurpose;
3750 if (T->isPointerTy())
3751 return AK_GeneralPurpose;
3752 return AK_Memory;
3753 }
3754
3755 // For VarArg functions, store the argument shadow in an ABI-specific format
3756 // that corresponds to va_list layout.
3757 // We do this because Clang lowers va_arg in the frontend, and this pass
3758 // only sees the low level code that deals with va_list internals.
3759 // A much easier alternative (provided that Clang emits va_arg instructions)
3760 // would have been to associate each live instance of va_list with a copy of
3761 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
3762 // order.
3763 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3764 unsigned GpOffset = 0;
3765 unsigned FpOffset = AMD64GpEndOffset;
3766 unsigned OverflowOffset = AMD64FpEndOffset;
3767 const DataLayout &DL = F.getParent()->getDataLayout();
3768 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3769 ArgIt != End; ++ArgIt) {
3770 Value *A = *ArgIt;
3771 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3772 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3773 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
3774 if (IsByVal) {
3775 // ByVal arguments always go to the overflow area.
3776 // Fixed arguments passed through the overflow area will be stepped
3777 // over by va_start, so don't count them towards the offset.
3778 if (IsFixed)
3779 continue;
3780 assert(A->getType()->isPointerTy())((A->getType()->isPointerTy()) ? static_cast<void>
(0) : __assert_fail ("A->getType()->isPointerTy()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3780, __PRETTY_FUNCTION__))
;
3781 Type *RealTy = A->getType()->getPointerElementType();
3782 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
3783 Value *ShadowBase = getShadowPtrForVAArgument(
3784 RealTy, IRB, OverflowOffset, alignTo(ArgSize, 8));
3785 Value *OriginBase = nullptr;
3786 if (MS.TrackOrigins)
3787 OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset);
3788 OverflowOffset += alignTo(ArgSize, 8);
3789 if (!ShadowBase)
3790 continue;
3791 Value *ShadowPtr, *OriginPtr;
3792 std::tie(ShadowPtr, OriginPtr) =
3793 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
3794 /*isStore*/ false);
3795
3796 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
3797 kShadowTLSAlignment, ArgSize);
3798 if (MS.TrackOrigins)
3799 IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
3800 kShadowTLSAlignment, ArgSize);
3801 } else {
3802 ArgKind AK = classifyArgument(A);
3803 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
3804 AK = AK_Memory;
3805 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
3806 AK = AK_Memory;
3807 Value *ShadowBase, *OriginBase = nullptr;
3808 switch (AK) {
3809 case AK_GeneralPurpose:
3810 ShadowBase =
3811 getShadowPtrForVAArgument(A->getType(), IRB, GpOffset, 8);
3812 if (MS.TrackOrigins)
3813 OriginBase =
3814 getOriginPtrForVAArgument(A->getType(), IRB, GpOffset);
3815 GpOffset += 8;
3816 break;
3817 case AK_FloatingPoint:
3818 ShadowBase =
3819 getShadowPtrForVAArgument(A->getType(), IRB, FpOffset, 16);
3820 if (MS.TrackOrigins)
3821 OriginBase =
3822 getOriginPtrForVAArgument(A->getType(), IRB, FpOffset);
3823 FpOffset += 16;
3824 break;
3825 case AK_Memory:
3826 if (IsFixed)
3827 continue;
3828 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3829 ShadowBase =
3830 getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, 8);
3831 if (MS.TrackOrigins)
3832 OriginBase =
3833 getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3834 OverflowOffset += alignTo(ArgSize, 8);
3835 }
3836 // Take fixed arguments into account for GpOffset and FpOffset,
3837 // but don't actually store shadows for them.
3838 // TODO(glider): don't call get*PtrForVAArgument() for them.
3839 if (IsFixed)
3840 continue;
3841 if (!ShadowBase)
3842 continue;
3843 Value *Shadow = MSV.getShadow(A);
3844 IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment);
3845 if (MS.TrackOrigins) {
3846 Value *Origin = MSV.getOrigin(A);
3847 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
3848 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
3849 std::max(kShadowTLSAlignment, kMinOriginAlignment));
3850 }
3851 }
3852 }
3853 Constant *OverflowSize =
3854 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
3855 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3856 }
3857
3858 /// Compute the shadow address for a given va_arg.
3859 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3860 unsigned ArgOffset, unsigned ArgSize) {
3861 // Make sure we don't overflow __msan_va_arg_tls.
3862 if (ArgOffset + ArgSize > kParamTLSSize)
3863 return nullptr;
3864 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3865 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3866 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3867 "_msarg_va_s");
3868 }
3869
3870 /// Compute the origin address for a given va_arg.
3871 Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) {
3872 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
3873 // getOriginPtrForVAArgument() is always called after
3874 // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
3875 // overflow.
3876 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3877 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
3878 "_msarg_va_o");
3879 }
3880
3881 void unpoisonVAListTagForInst(IntrinsicInst &I) {
3882 IRBuilder<> IRB(&I);
3883 Value *VAListTag = I.getArgOperand(0);
3884 Value *ShadowPtr, *OriginPtr;
3885 unsigned Alignment = 8;
3886 std::tie(ShadowPtr, OriginPtr) =
3887 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
3888 /*isStore*/ true);
3889
3890 // Unpoison the whole __va_list_tag.
3891 // FIXME: magic ABI constants.
3892 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3893 /* size */ 24, Alignment, false);
3894 // We shouldn't need to zero out the origins, as they're only checked for
3895 // nonzero shadow.
3896 }
3897
3898 void visitVAStartInst(VAStartInst &I) override {
3899 if (F.getCallingConv() == CallingConv::Win64)
3900 return;
3901 VAStartInstrumentationList.push_back(&I);
3902 unpoisonVAListTagForInst(I);
3903 }
3904
3905 void visitVACopyInst(VACopyInst &I) override {
3906 if (F.getCallingConv() == CallingConv::Win64) return;
3907 unpoisonVAListTagForInst(I);
3908 }
3909
3910 void finalizeInstrumentation() override {
3911 assert(!VAArgOverflowSize && !VAArgTLSCopy &&((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3912, __PRETTY_FUNCTION__))
3912 "finalizeInstrumentation called twice")((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3912, __PRETTY_FUNCTION__))
;
3913 if (!VAStartInstrumentationList.empty()) {
3914 // If there is a va_start in this function, make a backup copy of
3915 // va_arg_tls somewhere in the function entry block.
3916 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3917 VAArgOverflowSize =
3918 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
3919 Value *CopySize =
3920 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
3921 VAArgOverflowSize);
3922 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3923 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3924 if (MS.TrackOrigins) {
3925 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3926 IRB.CreateMemCpy(VAArgTLSOriginCopy, 8, MS.VAArgOriginTLS, 8, CopySize);
3927 }
3928 }
3929
3930 // Instrument va_start.
3931 // Copy va_list shadow from the backup copy of the TLS contents.
3932 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3933 CallInst *OrigInst = VAStartInstrumentationList[i];
3934 IRBuilder<> IRB(OrigInst->getNextNode());
3935 Value *VAListTag = OrigInst->getArgOperand(0);
3936
3937 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
3938 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
3939 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3940 ConstantInt::get(MS.IntptrTy, 16)),
3941 PointerType::get(RegSaveAreaPtrTy, 0));
3942 Value *RegSaveAreaPtr =
3943 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
3944 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
3945 unsigned Alignment = 16;
3946 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
3947 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3948 Alignment, /*isStore*/ true);
3949 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
3950 AMD64FpEndOffset);
3951 if (MS.TrackOrigins)
3952 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
3953 Alignment, AMD64FpEndOffset);
3954 Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
3955 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
3956 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3957 ConstantInt::get(MS.IntptrTy, 8)),
3958 PointerType::get(OverflowArgAreaPtrTy, 0));
3959 Value *OverflowArgAreaPtr =
3960 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
3961 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
3962 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
3963 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
3964 Alignment, /*isStore*/ true);
3965 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
3966 AMD64FpEndOffset);
3967 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
3968 VAArgOverflowSize);
3969 if (MS.TrackOrigins) {
3970 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
3971 AMD64FpEndOffset);
3972 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
3973 VAArgOverflowSize);
3974 }
3975 }
3976 }
3977};
3978
3979/// MIPS64-specific implementation of VarArgHelper.
3980struct VarArgMIPS64Helper : public VarArgHelper {
3981 Function &F;
3982 MemorySanitizer &MS;
3983 MemorySanitizerVisitor &MSV;
3984 Value *VAArgTLSCopy = nullptr;
3985 Value *VAArgSize = nullptr;
3986
3987 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3988
3989 VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
3990 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
3991
3992 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3993 unsigned VAArgOffset = 0;
3994 const DataLayout &DL = F.getParent()->getDataLayout();
3995 for (CallSite::arg_iterator ArgIt = CS.arg_begin() +
3996 CS.getFunctionType()->getNumParams(), End = CS.arg_end();
3997 ArgIt != End; ++ArgIt) {
3998 Triple TargetTriple(F.getParent()->getTargetTriple());
3999 Value *A = *ArgIt;
4000 Value *Base;
4001 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4002 if (TargetTriple.getArch() == Triple::mips64) {
4003 // Adjusting the shadow for argument with size < 8 to match the placement
4004 // of bits in big endian system
4005 if (ArgSize < 8)
4006 VAArgOffset += (8 - ArgSize);
4007 }
4008 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize);
4009 VAArgOffset += ArgSize;
4010 VAArgOffset = alignTo(VAArgOffset, 8);
4011 if (!Base)
4012 continue;
4013 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4014 }
4015
4016 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
4017 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4018 // a new class member i.e. it is the total size of all VarArgs.
4019 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4020 }
4021
4022 /// Compute the shadow address for a given va_arg.
4023 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4024 unsigned ArgOffset, unsigned ArgSize) {
4025 // Make sure we don't overflow __msan_va_arg_tls.
4026 if (ArgOffset + ArgSize > kParamTLSSize)
4027 return nullptr;
4028 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4029 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4030 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4031 "_msarg");
4032 }
4033
4034 void visitVAStartInst(VAStartInst &I) override {
4035 IRBuilder<> IRB(&I);
4036 VAStartInstrumentationList.push_back(&I);
4037 Value *VAListTag = I.getArgOperand(0);
4038 Value *ShadowPtr, *OriginPtr;
4039 unsigned Alignment = 8;
4040 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4041 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4042 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4043 /* size */ 8, Alignment, false);
4044 }
4045
4046 void visitVACopyInst(VACopyInst &I) override {
4047 IRBuilder<> IRB(&I);
4048 VAStartInstrumentationList.push_back(&I);
4049 Value *VAListTag = I.getArgOperand(0);
4050 Value *ShadowPtr, *OriginPtr;
4051 unsigned Alignment = 8;
4052 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4053 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4054 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4055 /* size */ 8, Alignment, false);
4056 }
4057
4058 void finalizeInstrumentation() override {
4059 assert(!VAArgSize && !VAArgTLSCopy &&((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4060, __PRETTY_FUNCTION__))
4060 "finalizeInstrumentation called twice")((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4060, __PRETTY_FUNCTION__))
;
4061 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
4062 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4063 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4064 VAArgSize);
4065
4066 if (!VAStartInstrumentationList.empty()) {
4067 // If there is a va_start in this function, make a backup copy of
4068 // va_arg_tls somewhere in the function entry block.
4069 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4070 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
4071 }
4072
4073 // Instrument va_start.
4074 // Copy va_list shadow from the backup copy of the TLS contents.
4075 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4076 CallInst *OrigInst = VAStartInstrumentationList[i];
4077 IRBuilder<> IRB(OrigInst->getNextNode());
4078 Value *VAListTag = OrigInst->getArgOperand(0);
4079 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4080 Value *RegSaveAreaPtrPtr =
4081 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4082 PointerType::get(RegSaveAreaPtrTy, 0));
4083 Value *RegSaveAreaPtr =
4084 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4085 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4086 unsigned Alignment = 8;
4087 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4088 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4089 Alignment, /*isStore*/ true);
4090 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4091 CopySize);
4092 }
4093 }
4094};
4095
4096/// AArch64-specific implementation of VarArgHelper.
4097struct VarArgAArch64Helper : public VarArgHelper {
4098 static const unsigned kAArch64GrArgSize = 64;
4099 static const unsigned kAArch64VrArgSize = 128;
4100
4101 static const unsigned AArch64GrBegOffset = 0;
4102 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
4103 // Make VR space aligned to 16 bytes.
4104 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
4105 static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
4106 + kAArch64VrArgSize;
4107 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
4108
4109 Function &F;
4110 MemorySanitizer &MS;
4111 MemorySanitizerVisitor &MSV;
4112 Value *VAArgTLSCopy = nullptr;
4113 Value *VAArgOverflowSize = nullptr;
4114
4115 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4116
4117 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4118
4119 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
4120 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4121
4122 ArgKind classifyArgument(Value* arg) {
4123 Type *T = arg->getType();
4124 if (T->isFPOrFPVectorTy())
4125 return AK_FloatingPoint;
4126 if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4127 || (T->isPointerTy()))
4128 return AK_GeneralPurpose;
4129 return AK_Memory;
4130 }
4131
4132 // The instrumentation stores the argument shadow in a non ABI-specific
4133 // format because it does not know which argument is named (since Clang,
4134 // like x86_64 case, lowers the va_args in the frontend and this pass only
4135 // sees the low level code that deals with va_list internals).
4136 // The first seven GR registers are saved in the first 56 bytes of the
4137 // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
4138 // the remaining arguments.
4139 // Using constant offset within the va_arg TLS array allows fast copy
4140 // in the finalize instrumentation.
4141 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
4142 unsigned GrOffset = AArch64GrBegOffset;
4143 unsigned VrOffset = AArch64VrBegOffset;
4144 unsigned OverflowOffset = AArch64VAEndOffset;
4145
4146 const DataLayout &DL = F.getParent()->getDataLayout();
4147 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
4148 ArgIt != End; ++ArgIt) {
4149 Value *A = *ArgIt;
4150 unsigned ArgNo = CS.getArgumentNo(ArgIt);
4151 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
4152 ArgKind AK = classifyArgument(A);
4153 if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
4154 AK = AK_Memory;
4155 if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
4156 AK = AK_Memory;
4157 Value *Base;
4158 switch (AK) {
4159 case AK_GeneralPurpose:
4160 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset, 8);
4161 GrOffset += 8;
4162 break;
4163 case AK_FloatingPoint:
4164 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset, 8);
4165 VrOffset += 16;
4166 break;
4167 case AK_Memory:
4168 // Don't count fixed arguments in the overflow area - va_start will
4169 // skip right over them.
4170 if (IsFixed)
4171 continue;
4172 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4173 Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset,
4174 alignTo(ArgSize, 8));
4175 OverflowOffset += alignTo(ArgSize, 8);
4176 break;
4177 }
4178 // Count Gp/Vr fixed arguments to their respective offsets, but don't
4179 // bother to actually store a shadow.
4180 if (IsFixed)
4181 continue;
4182 if (!Base)
4183 continue;
4184 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4185 }
4186 Constant *OverflowSize =
4187 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
4188 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4189 }
4190
4191 /// Compute the shadow address for a given va_arg.
4192 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4193 unsigned ArgOffset, unsigned ArgSize) {
4194 // Make sure we don't overflow __msan_va_arg_tls.
4195 if (ArgOffset + ArgSize > kParamTLSSize)
4196 return nullptr;
4197 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4198 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4199 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4200 "_msarg");
4201 }
4202
4203 void visitVAStartInst(VAStartInst &I) override {
4204 IRBuilder<> IRB(&I);
4205 VAStartInstrumentationList.push_back(&I);
4206 Value *VAListTag = I.getArgOperand(0);
4207 Value *ShadowPtr, *OriginPtr;
4208 unsigned Alignment = 8;
4209 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4210 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4211 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4212 /* size */ 32, Alignment, false);
4213 }
4214
4215 void visitVACopyInst(VACopyInst &I) override {
4216 IRBuilder<> IRB(&I);
4217 VAStartInstrumentationList.push_back(&I);
4218 Value *VAListTag = I.getArgOperand(0);
4219 Value *ShadowPtr, *OriginPtr;
4220 unsigned Alignment = 8;
4221 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4222 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4223 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4224 /* size */ 32, Alignment, false);
4225 }
4226
4227 // Retrieve a va_list field of 'void*' size.
4228 Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4229 Value *SaveAreaPtrPtr =
4230 IRB.CreateIntToPtr(
4231 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4232 ConstantInt::get(MS.IntptrTy, offset)),
4233 Type::getInt64PtrTy(*MS.C));
4234 return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
4235 }
4236
4237 // Retrieve a va_list field of 'int' size.
4238 Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4239 Value *SaveAreaPtr =
4240 IRB.CreateIntToPtr(
4241 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4242 ConstantInt::get(MS.IntptrTy, offset)),
4243 Type::getInt32PtrTy(*MS.C));
4244 Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
4245 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
4246 }
4247
4248 void finalizeInstrumentation() override {
4249 assert(!VAArgOverflowSize && !VAArgTLSCopy &&((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4250, __PRETTY_FUNCTION__))
4250 "finalizeInstrumentation called twice")((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4250, __PRETTY_FUNCTION__))
;
4251 if (!VAStartInstrumentationList.empty()) {
4252 // If there is a va_start in this function, make a backup copy of
4253 // va_arg_tls somewhere in the function entry block.
4254 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
4255 VAArgOverflowSize =
4256 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4257 Value *CopySize =
4258 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
4259 VAArgOverflowSize);
4260 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4261 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
4262 }
4263
4264 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
4265 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
4266
4267 // Instrument va_start, copy va_list shadow from the backup copy of
4268 // the TLS contents.
4269 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4270 CallInst *OrigInst = VAStartInstrumentationList[i];
4271 IRBuilder<> IRB(OrigInst->getNextNode());
4272
4273 Value *VAListTag = OrigInst->getArgOperand(0);
4274
4275 // The variadic ABI for AArch64 creates two areas to save the incoming
4276 // argument registers (one for 64-bit general register xn-x7 and another
4277 // for 128-bit FP/SIMD vn-v7).
4278 // We need then to propagate the shadow arguments on both regions
4279 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
4280 // The remaning arguments are saved on shadow for 'va::stack'.
4281 // One caveat is it requires only to propagate the non-named arguments,
4282 // however on the call site instrumentation 'all' the arguments are
4283 // saved. So to copy the shadow values from the va_arg TLS array
4284 // we need to adjust the offset for both GR and VR fields based on
4285 // the __{gr,vr}_offs value (since they are stores based on incoming
4286 // named arguments).
4287
4288 // Read the stack pointer from the va_list.
4289 Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
4290
4291 // Read both the __gr_top and __gr_off and add them up.
4292 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
4293 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
4294
4295 Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
4296
4297 // Read both the __vr_top and __vr_off and add them up.
4298 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
4299 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
4300
4301 Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
4302
4303 // It does not know how many named arguments is being used and, on the
4304 // callsite all the arguments were saved. Since __gr_off is defined as
4305 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
4306 // argument by ignoring the bytes of shadow from named arguments.
4307 Value *GrRegSaveAreaShadowPtrOff =
4308 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
4309
4310 Value *GrRegSaveAreaShadowPtr =
4311 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4312 /*Alignment*/ 8, /*isStore*/ true)
4313 .first;
4314
4315 Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4316 GrRegSaveAreaShadowPtrOff);
4317 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
4318
4319 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, 8, GrSrcPtr, 8, GrCopySize);
4320
4321 // Again, but for FP/SIMD values.
4322 Value *VrRegSaveAreaShadowPtrOff =
4323 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
4324
4325 Value *VrRegSaveAreaShadowPtr =
4326 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4327 /*Alignment*/ 8, /*isStore*/ true)
4328 .first;
4329
4330 Value *VrSrcPtr = IRB.CreateInBoundsGEP(
4331 IRB.getInt8Ty(),
4332 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4333 IRB.getInt32(AArch64VrBegOffset)),
4334 VrRegSaveAreaShadowPtrOff);
4335 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
4336
4337 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, 8, VrSrcPtr, 8, VrCopySize);
4338
4339 // And finally for remaining arguments.
4340 Value *StackSaveAreaShadowPtr =
4341 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
4342 /*Alignment*/ 16, /*isStore*/ true)
4343 .first;
4344
4345 Value *StackSrcPtr =
4346 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4347 IRB.getInt32(AArch64VAEndOffset));
4348
4349 IRB.CreateMemCpy(StackSaveAreaShadowPtr, 16, StackSrcPtr, 16,
4350 VAArgOverflowSize);
4351 }
4352 }
4353};
4354
4355/// PowerPC64-specific implementation of VarArgHelper.
4356struct VarArgPowerPC64Helper : public VarArgHelper {
4357 Function &F;
4358 MemorySanitizer &MS;
4359 MemorySanitizerVisitor &MSV;
4360 Value *VAArgTLSCopy = nullptr;
4361 Value *VAArgSize = nullptr;
4362
4363 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4364
4365 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
4366 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4367
4368 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
4369 // For PowerPC, we need to deal with alignment of stack arguments -
4370 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
4371 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
4372 // and QPX vectors are aligned to 32 bytes. For that reason, we
4373 // compute current offset from stack pointer (which is always properly
4374 // aligned), and offset for the first vararg, then subtract them.
4375 unsigned VAArgBase;
4376 Triple TargetTriple(F.getParent()->getTargetTriple());
4377 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
4378 // and 32 bytes for ABIv2. This is usually determined by target
4379 // endianness, but in theory could be overriden by function attribute.
4380 // For simplicity, we ignore it here (it'd only matter for QPX vectors).
4381 if (TargetTriple.getArch() == Triple::ppc64)
4382 VAArgBase = 48;
4383 else
4384 VAArgBase = 32;
4385 unsigned VAArgOffset = VAArgBase;
4386 const DataLayout &DL = F.getParent()->getDataLayout();
4387 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
4388 ArgIt != End; ++ArgIt) {
4389 Value *A = *ArgIt;
4390 unsigned ArgNo = CS.getArgumentNo(ArgIt);
4391 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
4392 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
4393 if (IsByVal) {
4394 assert(A->getType()->isPointerTy())((A->getType()->isPointerTy()) ? static_cast<void>
(0) : __assert_fail ("A->getType()->isPointerTy()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4394, __PRETTY_FUNCTION__))
;
4395 Type *RealTy = A->getType()->getPointerElementType();
4396 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4397 uint64_t ArgAlign = CS.getParamAlignment(ArgNo);
4398 if (ArgAlign < 8)
4399 ArgAlign = 8;
4400 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4401 if (!IsFixed) {
4402 Value *Base = getShadowPtrForVAArgument(
4403 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
4404 if (Base) {
4405 Value *AShadowPtr, *AOriginPtr;
4406 std::tie(AShadowPtr, AOriginPtr) =
4407 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
4408 kShadowTLSAlignment, /*isStore*/ false);
4409
4410 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
4411 kShadowTLSAlignment, ArgSize);
4412 }
4413 }
4414 VAArgOffset += alignTo(ArgSize, 8);
4415 } else {
4416 Value *Base;
4417 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4418 uint64_t ArgAlign = 8;
4419 if (A->getType()->isArrayTy()) {
4420 // Arrays are aligned to element size, except for long double
4421 // arrays, which are aligned to 8 bytes.
4422 Type *ElementTy = A->getType()->getArrayElementType();
4423 if (!ElementTy->isPPC_FP128Ty())
4424 ArgAlign = DL.getTypeAllocSize(ElementTy);
4425 } else if (A->getType()->isVectorTy()) {
4426 // Vectors are naturally aligned.
4427 ArgAlign = DL.getTypeAllocSize(A->getType());
4428 }
4429 if (ArgAlign < 8)
4430 ArgAlign = 8;
4431 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4432 if (DL.isBigEndian()) {
4433 // Adjusting the shadow for argument with size < 8 to match the placement
4434 // of bits in big endian system
4435 if (ArgSize < 8)
4436 VAArgOffset += (8 - ArgSize);
4437 }
4438 if (!IsFixed) {
4439 Base = getShadowPtrForVAArgument(A->getType(), IRB,
4440 VAArgOffset - VAArgBase, ArgSize);
4441 if (Base)
4442 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4443 }
4444 VAArgOffset += ArgSize;
4445 VAArgOffset = alignTo(VAArgOffset, 8);
4446 }
4447 if (IsFixed)
4448 VAArgBase = VAArgOffset;
4449 }
4450
4451 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
4452 VAArgOffset - VAArgBase);
4453 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4454 // a new class member i.e. it is the total size of all VarArgs.
4455 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4456 }
4457
4458 /// Compute the shadow address for a given va_arg.
4459 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4460 unsigned ArgOffset, unsigned ArgSize) {
4461 // Make sure we don't overflow __msan_va_arg_tls.
4462 if (ArgOffset + ArgSize > kParamTLSSize)
4463 return nullptr;
4464 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4465 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4466 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4467 "_msarg");
4468 }
4469
4470 void visitVAStartInst(VAStartInst &I) override {
4471 IRBuilder<> IRB(&I);
4472 VAStartInstrumentationList.push_back(&I);
4473 Value *VAListTag = I.getArgOperand(0);
4474 Value *ShadowPtr, *OriginPtr;
4475 unsigned Alignment = 8;
4476 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4477 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4478 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4479 /* size */ 8, Alignment, false);
4480 }
4481
4482 void visitVACopyInst(VACopyInst &I) override {
4483 IRBuilder<> IRB(&I);
4484 Value *VAListTag = I.getArgOperand(0);
4485 Value *ShadowPtr, *OriginPtr;
4486 unsigned Alignment = 8;
4487 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4488 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4489 // Unpoison the whole __va_list_tag.
4490 // FIXME: magic ABI constants.
4491 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4492 /* size */ 8, Alignment, false);
4493 }
4494
4495 void finalizeInstrumentation() override {
4496 assert(!VAArgSize && !VAArgTLSCopy &&((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4497, __PRETTY_FUNCTION__))
4497 "finalizeInstrumentation called twice")((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4497, __PRETTY_FUNCTION__))
;
4498 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
4499 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4500 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4501 VAArgSize);
4502
4503 if (!VAStartInstrumentationList.empty()) {
4504 // If there is a va_start in this function, make a backup copy of
4505 // va_arg_tls somewhere in the function entry block.
4506 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4507 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
4508 }
4509
4510 // Instrument va_start.
4511 // Copy va_list shadow from the backup copy of the TLS contents.
4512 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4513 CallInst *OrigInst = VAStartInstrumentationList[i];
4514 IRBuilder<> IRB(OrigInst->getNextNode());
4515 Value *VAListTag = OrigInst->getArgOperand(0);
4516 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4517 Value *RegSaveAreaPtrPtr =
4518 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4519 PointerType::get(RegSaveAreaPtrTy, 0));
4520 Value *RegSaveAreaPtr =
4521 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4522 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4523 unsigned Alignment = 8;
4524 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4525 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4526 Alignment, /*isStore*/ true);
4527 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4528 CopySize);
4529 }
4530 }
4531};
4532
4533/// A no-op implementation of VarArgHelper.
4534struct VarArgNoOpHelper : public VarArgHelper {
4535 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
4536 MemorySanitizerVisitor &MSV) {}
4537
4538 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {}
4539
4540 void visitVAStartInst(VAStartInst &I) override {}
4541
4542 void visitVACopyInst(VACopyInst &I) override {}
4543
4544 void finalizeInstrumentation() override {}
4545};
4546
4547} // end anonymous namespace
4548
4549static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
4550 MemorySanitizerVisitor &Visitor) {
4551 // VarArg handling is only implemented on AMD64. False positives are possible
4552 // on other platforms.
4553 Triple TargetTriple(Func.getParent()->getTargetTriple());
4554 if (TargetTriple.getArch() == Triple::x86_64)
4555 return new VarArgAMD64Helper(Func, Msan, Visitor);
4556 else if (TargetTriple.isMIPS64())
4557 return new VarArgMIPS64Helper(Func, Msan, Visitor);
4558 else if (TargetTriple.getArch() == Triple::aarch64)
4559 return new VarArgAArch64Helper(Func, Msan, Visitor);
4560 else if (TargetTriple.getArch() == Triple::ppc64 ||
4561 TargetTriple.getArch() == Triple::ppc64le)
4562 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
4563 else
4564 return new VarArgNoOpHelper(Func, Msan, Visitor);
4565}
4566
4567bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
4568 if (!CompileKernel && (&F == MsanCtorFunction))
4569 return false;
4570 MemorySanitizerVisitor Visitor(F, *this, TLI);
4571
4572 // Clear out readonly/readnone attributes.
4573 AttrBuilder B;
4574 B.addAttribute(Attribute::ReadOnly)
4575 .addAttribute(Attribute::ReadNone);
4576 F.removeAttributes(AttributeList::FunctionIndex, B);
4577
4578 return Visitor.runOnFunction();
4579}