181#include "llvm/IR/IntrinsicsX86.h"
210#define DEBUG_TYPE "msan"
213 "Controls which checks to insert");
231 "msan-track-origins",
236 cl::desc(
"keep going after reporting a UMR"),
245 "msan-poison-stack-with-call",
250 "msan-poison-stack-pattern",
251 cl::desc(
"poison uninitialized stack variables with the given pattern"),
256 cl::desc(
"Print name of local stack variable"),
265 cl::desc(
"propagate shadow through ICmpEQ and ICmpNE"),
270 cl::desc(
"exact handling of relational integer ICmp"),
274 "msan-handle-lifetime-intrinsics",
276 "when possible, poison scoped variables at the beginning of the scope "
277 "(slower, but more precise)"),
288 "msan-handle-asm-conservative",
299 "msan-check-access-address",
300 cl::desc(
"report accesses through a pointer which has poisoned shadow"),
305 cl::desc(
"check arguments and return values at function call boundaries"),
309 "msan-dump-strict-instructions",
310 cl::desc(
"print out instructions with default strict semantics"),
314 "msan-instrumentation-with-call-threshold",
316 "If the function being instrumented requires more than "
317 "this number of checks and origin stores, use callbacks instead of "
318 "inline checks (-1 means never use callbacks)."),
323 cl::desc(
"Enable KernelMemorySanitizer instrumentation"),
333 cl::desc(
"Insert checks for constant shadow values"),
340 cl::desc(
"Place MSan constructors in comdat sections"),
346 cl::desc(
"Define custom MSan AndMask"),
350 cl::desc(
"Define custom MSan XorMask"),
354 cl::desc(
"Define custom MSan ShadowBase"),
358 cl::desc(
"Define custom MSan OriginBase"),
363 cl::desc(
"Define threshold for number of checks per "
364 "debug location to force origin update."),
376struct MemoryMapParams {
383struct PlatformMemoryMapParams {
384 const MemoryMapParams *bits32;
385 const MemoryMapParams *bits64;
531class MemorySanitizer {
540 MemorySanitizer(MemorySanitizer &&) =
delete;
541 MemorySanitizer &operator=(MemorySanitizer &&) =
delete;
542 MemorySanitizer(
const MemorySanitizer &) =
delete;
543 MemorySanitizer &operator=(
const MemorySanitizer &) =
delete;
548 friend struct MemorySanitizerVisitor;
549 friend struct VarArgHelperBase;
550 friend struct VarArgAMD64Helper;
551 friend struct VarArgMIPS64Helper;
552 friend struct VarArgAArch64Helper;
553 friend struct VarArgPowerPC64Helper;
554 friend struct VarArgSystemZHelper;
556 void initializeModule(
Module &M);
561 template <
typename... ArgsTy>
588 Value *ParamOriginTLS;
594 Value *RetvalOriginTLS;
600 Value *VAArgOriginTLS;
603 Value *VAArgOverflowSizeTLS;
606 bool CallbacksInitialized =
false;
651 Value *MsanMetadataAlloca;
657 const MemoryMapParams *MapParams;
661 MemoryMapParams CustomMapParams;
666 MDNode *OriginStoreWeights;
669void insertModuleCtor(
Module &M) {
697 Recover(getOptOrDefault(
ClKeepGoing, Kernel || R)),
712 MemorySanitizer Msan(*
F.getParent(),
Options);
731 OS, MapClassName2PassName);
738 OS <<
"eager-checks;";
739 OS <<
"track-origins=" <<
Options.TrackOrigins;
755template <
typename... ArgsTy>
763 std::forward<ArgsTy>(Args)...);
766 return M.getOrInsertFunction(
Name, MsanMetadata,
767 std::forward<ArgsTy>(Args)...);
776 RetvalOriginTLS =
nullptr;
778 ParamOriginTLS =
nullptr;
780 VAArgOriginTLS =
nullptr;
781 VAArgOverflowSizeTLS =
nullptr;
783 WarningFn =
M.getOrInsertFunction(
"__msan_warning",
785 IRB.getVoidTy(), IRB.getInt32Ty());
796 MsanGetContextStateFn =
M.getOrInsertFunction(
802 for (
int ind = 0, size = 1; ind < 4; ind++,
size <<= 1) {
803 std::string name_load =
804 "__msan_metadata_ptr_for_load_" + std::to_string(size);
805 std::string name_store =
806 "__msan_metadata_ptr_for_store_" + std::to_string(size);
807 MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction(
809 MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction(
813 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
816 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
817 M,
"__msan_metadata_ptr_for_store_n",
821 MsanPoisonAllocaFn =
M.getOrInsertFunction(
822 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
823 MsanUnpoisonAllocaFn =
M.getOrInsertFunction(
824 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
828 return M.getOrInsertGlobal(
Name, Ty, [&] {
830 nullptr,
Name,
nullptr,
843 StringRef WarningFnName = Recover ?
"__msan_warning_with_origin"
844 :
"__msan_warning_with_origin_noreturn";
845 WarningFn =
M.getOrInsertFunction(WarningFnName,
847 IRB.getVoidTy(), IRB.getInt32Ty());
850 Recover ?
"__msan_warning" :
"__msan_warning_noreturn";
851 WarningFn =
M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
877 VAArgOverflowSizeTLS =
882 unsigned AccessSize = 1 << AccessSizeIndex;
883 std::string FunctionName =
"__msan_maybe_warning_" + itostr(AccessSize);
884 MaybeWarningFn[AccessSizeIndex] =
M.getOrInsertFunction(
886 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
888 FunctionName =
"__msan_maybe_store_origin_" + itostr(AccessSize);
889 MaybeStoreOriginFn[AccessSizeIndex] =
M.getOrInsertFunction(
891 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
895 MsanSetAllocaOriginWithDescriptionFn =
896 M.getOrInsertFunction(
"__msan_set_alloca_origin_with_descr",
897 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
898 MsanSetAllocaOriginNoDescriptionFn =
899 M.getOrInsertFunction(
"__msan_set_alloca_origin_no_descr",
900 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
901 MsanPoisonStackFn =
M.getOrInsertFunction(
"__msan_poison_stack",
902 IRB.getVoidTy(), PtrTy, IntptrTy);
908 if (CallbacksInitialized)
914 MsanChainOriginFn =
M.getOrInsertFunction(
915 "__msan_chain_origin",
918 MsanSetOriginFn =
M.getOrInsertFunction(
920 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
922 M.getOrInsertFunction(
"__msan_memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
924 M.getOrInsertFunction(
"__msan_memcpy", PtrTy, PtrTy, PtrTy, IntptrTy);
925 MemsetFn =
M.getOrInsertFunction(
"__msan_memset",
927 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
929 MsanInstrumentAsmStoreFn =
930 M.getOrInsertFunction(
"__msan_instrument_asm_store", IRB.getVoidTy(),
934 createKernelApi(M, TLI);
936 createUserspaceApi(M, TLI);
938 CallbacksInitialized =
true;
944 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
962void MemorySanitizer::initializeModule(
Module &M) {
963 auto &
DL =
M.getDataLayout();
965 TargetTriple =
Triple(
M.getTargetTriple());
967 bool ShadowPassed =
ClShadowBase.getNumOccurrences() > 0;
968 bool OriginPassed =
ClOriginBase.getNumOccurrences() > 0;
970 if (ShadowPassed || OriginPassed) {
975 MapParams = &CustomMapParams;
977 switch (TargetTriple.getOS()) {
979 switch (TargetTriple.getArch()) {
994 switch (TargetTriple.getArch()) {
1003 switch (TargetTriple.getArch()) {
1037 C = &(
M.getContext());
1039 IntptrTy = IRB.getIntPtrTy(
DL);
1040 OriginTy = IRB.getInt32Ty();
1041 PtrTy = IRB.getPtrTy();
1046 if (!CompileKernel) {
1048 M.getOrInsertGlobal(
"__msan_track_origins", IRB.getInt32Ty(), [&] {
1049 return new GlobalVariable(
1050 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1051 IRB.getInt32(TrackOrigins),
"__msan_track_origins");
1055 M.getOrInsertGlobal(
"__msan_keep_going", IRB.getInt32Ty(), [&] {
1056 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1057 GlobalValue::WeakODRLinkage,
1058 IRB.getInt32(Recover),
"__msan_keep_going");
1073struct VarArgHelper {
1074 virtual ~VarArgHelper() =
default;
1089 virtual void finalizeInstrumentation() = 0;
1092struct MemorySanitizerVisitor;
1097 MemorySanitizerVisitor &Visitor);
1104 if (TypeSizeFixed <= 8)
1113class NextNodeIRBuilder :
public IRBuilder<> {
1126struct MemorySanitizerVisitor :
public InstVisitor<MemorySanitizerVisitor> {
1128 MemorySanitizer &MS;
1131 std::unique_ptr<VarArgHelper> VAHelper;
1138 bool PropagateShadow;
1142 struct ShadowOriginAndInsertPoint {
1148 : Shadow(S), Origin(
O), OrigIns(
I) {}
1156 int64_t SplittableBlocksCount = 0;
1158 MemorySanitizerVisitor(
Function &
F, MemorySanitizer &MS,
1161 bool SanitizeFunction =
1163 InsertChecks = SanitizeFunction;
1164 PropagateShadow = SanitizeFunction;
1174 MS.initializeCallbacks(*
F.getParent(), TLI);
1175 FnPrologueEnd =
IRBuilder<>(
F.getEntryBlock().getFirstNonPHI())
1178 if (MS.CompileKernel) {
1180 insertKmsanPrologue(IRB);
1184 <<
"MemorySanitizer is not inserting checks into '"
1185 <<
F.getName() <<
"'\n");
1188 bool instrumentWithCalls(
Value *V) {
1190 if (isa<Constant>(V))
1193 ++SplittableBlocksCount;
1199 return I.getParent() == FnPrologueEnd->
getParent() &&
1200 (&
I == FnPrologueEnd ||
I.comesBefore(FnPrologueEnd));
1208 if (MS.TrackOrigins <= 1)
1210 return IRB.
CreateCall(MS.MsanChainOriginFn, V);
1215 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1227 const Align IntptrAlignment =
DL.getABITypeAlign(MS.IntptrTy);
1228 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1238 auto [InsertPt,
Index] =
1250 Align CurrentAlignment = Alignment;
1251 if (Alignment >= IntptrAlignment && IntptrSize >
kOriginSize) {
1252 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1253 Value *IntptrOriginPtr =
1255 for (
unsigned i = 0; i <
Size / IntptrSize; ++i) {
1260 CurrentAlignment = IntptrAlignment;
1277 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1278 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1286 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1295 if (instrumentWithCalls(ConvertedShadow) &&
1298 Value *ConvertedShadow2 =
1304 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1308 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1313 void materializeStores() {
1316 Value *Val =
SI->getValueOperand();
1318 Value *Shadow =
SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1319 Value *ShadowPtr, *OriginPtr;
1321 const Align Alignment =
SI->getAlign();
1323 std::tie(ShadowPtr, OriginPtr) =
1324 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
true);
1333 if (MS.TrackOrigins && !
SI->isAtomic())
1334 storeOrigin(IRB,
Addr, Shadow, getOrigin(Val), OriginPtr,
1341 if (MS.TrackOrigins < 2)
1344 if (LazyWarningDebugLocationCount.
empty())
1345 for (
const auto &
I : InstrumentationList)
1346 ++LazyWarningDebugLocationCount[
I.OrigIns->getDebugLoc()];
1360 if (
Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1362 auto NewDebugLoc = OI->getDebugLoc();
1369 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1370 Origin = updateOrigin(Origin, IRBOrigin);
1375 if (MS.CompileKernel || MS.TrackOrigins)
1389 if (instrumentWithCalls(ConvertedShadow) &&
1392 Value *ConvertedShadow2 =
1395 Fn, {ConvertedShadow2,
1396 MS.TrackOrigins && Origin ? Origin : (
Value *)IRB.
getInt32(0)});
1400 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1403 !MS.Recover, MS.ColdCallWeights);
1406 insertWarningFn(IRB, Origin);
1411 void materializeInstructionChecks(
1416 bool Combine = !MS.TrackOrigins;
1418 Value *Shadow =
nullptr;
1419 for (
const auto &ShadowData : InstructionChecks) {
1423 Value *ConvertedShadow = ShadowData.Shadow;
1425 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1432 insertWarningFn(IRB, ShadowData.Origin);
1442 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1447 Shadow = ConvertedShadow;
1451 Shadow = convertToBool(Shadow, IRB,
"_mscmp");
1452 ConvertedShadow = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1453 Shadow = IRB.
CreateOr(Shadow, ConvertedShadow,
"_msor");
1459 materializeOneCheck(IRB, Shadow,
nullptr);
1463 void materializeChecks() {
1465 [](
const ShadowOriginAndInsertPoint &L,
1466 const ShadowOriginAndInsertPoint &R) {
1467 return L.OrigIns <
R.OrigIns;
1470 for (
auto I = InstrumentationList.begin();
1471 I != InstrumentationList.end();) {
1473 std::find_if(
I + 1, InstrumentationList.end(),
1474 [L =
I->OrigIns](
const ShadowOriginAndInsertPoint &R) {
1475 return L != R.OrigIns;
1489 MS.ParamTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1490 {Zero, IRB.getInt32(0)},
"param_shadow");
1491 MS.RetvalTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1492 {Zero, IRB.getInt32(1)},
"retval_shadow");
1493 MS.VAArgTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1494 {Zero, IRB.getInt32(2)},
"va_arg_shadow");
1495 MS.VAArgOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1496 {Zero, IRB.getInt32(3)},
"va_arg_origin");
1497 MS.VAArgOverflowSizeTLS =
1498 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1499 {Zero, IRB.getInt32(4)},
"va_arg_overflow_size");
1500 MS.ParamOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1501 {Zero, IRB.getInt32(5)},
"param_origin");
1502 MS.RetvalOriginTLS =
1503 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1504 {Zero, IRB.getInt32(6)},
"retval_origin");
1506 MS.MsanMetadataAlloca = IRB.
CreateAlloca(MS.MsanMetadata, 0u);
1518 for (
PHINode *PN : ShadowPHINodes) {
1519 PHINode *PNS = cast<PHINode>(getShadow(PN));
1520 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1521 size_t NumValues = PN->getNumIncomingValues();
1522 for (
size_t v = 0;
v < NumValues;
v++) {
1523 PNS->
addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1525 PNO->
addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1529 VAHelper->finalizeInstrumentation();
1533 if (InstrumentLifetimeStart) {
1534 for (
auto Item : LifetimeStartList) {
1535 instrumentAlloca(*Item.second, Item.first);
1536 AllocaSet.
remove(Item.second);
1542 instrumentAlloca(*AI);
1545 materializeChecks();
1549 materializeStores();
1555 Type *getShadowTy(
Value *V) {
return getShadowTy(
V->getType()); }
1567 if (
VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1568 uint32_t EltSize =
DL.getTypeSizeInBits(VT->getElementType());
1570 VT->getElementCount());
1572 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1573 return ArrayType::get(getShadowTy(AT->getElementType()),
1574 AT->getNumElements());
1576 if (
StructType *ST = dyn_cast<StructType>(OrigTy)) {
1578 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1579 Elements.push_back(getShadowTy(
ST->getElementType(i)));
1581 LLVM_DEBUG(
dbgs() <<
"getShadowTy: " << *ST <<
" ===> " << *Res <<
"\n");
1597 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1599 if (Aggregator != FalseVal)
1600 Aggregator = IRB.
CreateOr(Aggregator, ShadowBool);
1602 Aggregator = ShadowBool;
1611 if (!
Array->getNumElements())
1615 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1619 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1620 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
1630 return collapseStructShadow(
Struct, V, IRB);
1631 if (
ArrayType *Array = dyn_cast<ArrayType>(
V->getType()))
1632 return collapseArrayShadow(Array, V, IRB);
1633 if (isa<VectorType>(
V->getType())) {
1634 if (isa<ScalableVectorType>(
V->getType()))
1637 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1645 Type *VTy =
V->getType();
1647 return convertToBool(convertShadowToScalar(V, IRB), IRB,
name);
1654 Type *ptrToIntPtrType(
Type *PtrTy)
const {
1655 if (
VectorType *VectTy = dyn_cast<VectorType>(PtrTy)) {
1656 return VectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1657 VectTy->getElementCount());
1663 Type *getPtrToShadowPtrType(
Type *IntPtrTy,
Type *ShadowTy)
const {
1664 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1665 return VectorType::get(
1666 getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1667 VectTy->getElementCount());
1669 assert(IntPtrTy == MS.IntptrTy);
1670 return PointerType::get(*MS.C, 0);
1674 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1676 VectTy->getElementCount(), constToIntPtr(VectTy->getElementType(),
C));
1678 assert(IntPtrTy == MS.IntptrTy);
1679 return ConstantInt::get(MS.IntptrTy,
C);
1690 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1693 if (
uint64_t AndMask = MS.MapParams->AndMask)
1694 OffsetLong = IRB.
CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1696 if (
uint64_t XorMask = MS.MapParams->XorMask)
1697 OffsetLong = IRB.
CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1709 std::pair<Value *, Value *>
1716 assert(VectTy->getElementType()->isPointerTy());
1718 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1719 Value *ShadowOffset = getShadowPtrOffset(
Addr, IRB);
1720 Value *ShadowLong = ShadowOffset;
1721 if (
uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1723 IRB.
CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1726 ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1728 Value *OriginPtr =
nullptr;
1729 if (MS.TrackOrigins) {
1730 Value *OriginLong = ShadowOffset;
1731 uint64_t OriginBase = MS.MapParams->OriginBase;
1732 if (OriginBase != 0)
1734 IRB.
CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1737 OriginLong = IRB.
CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1740 OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
1742 return std::make_pair(ShadowPtr, OriginPtr);
1745 template <
typename... ArgsTy>
1750 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1751 return IRB.
CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1754 return IRB.
CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1757 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(
Value *
Addr,
1761 Value *ShadowOriginPtrs;
1769 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1771 Value *SizeVal = ConstantInt::get(MS.IntptrTy,
Size);
1772 ShadowOriginPtrs = createMetadataCall(
1774 isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1781 return std::make_pair(ShadowPtr, OriginPtr);
1787 std::pair<Value *, Value *> getShadowOriginPtrKernel(
Value *
Addr,
1794 return getShadowOriginPtrKernelNoVec(
Addr, IRB, ShadowTy,
isStore);
1798 unsigned NumElements = cast<FixedVectorType>(VectTy)->getNumElements();
1799 Value *ShadowPtrs = ConstantInt::getNullValue(
1801 Value *OriginPtrs =
nullptr;
1802 if (MS.TrackOrigins)
1803 OriginPtrs = ConstantInt::getNullValue(
1805 for (
unsigned i = 0; i < NumElements; ++i) {
1808 auto [ShadowPtr, OriginPtr] =
1809 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy,
isStore);
1812 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1813 if (MS.TrackOrigins)
1815 OriginPtrs, OriginPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1817 return {ShadowPtrs, OriginPtrs};
1824 if (MS.CompileKernel)
1825 return getShadowOriginPtrKernel(
Addr, IRB, ShadowTy,
isStore);
1826 return getShadowOriginPtrUserspace(
Addr, IRB, ShadowTy, Alignment);
1841 if (!MS.TrackOrigins)
1855 Value *getOriginPtrForRetval() {
1857 return MS.RetvalOriginTLS;
1862 assert(!ShadowMap.
count(V) &&
"Values may only have one shadow");
1863 ShadowMap[
V] = PropagateShadow ? SV : getCleanShadow(V);
1868 if (!MS.TrackOrigins)
1870 assert(!OriginMap.
count(V) &&
"Values may only have one origin");
1871 LLVM_DEBUG(
dbgs() <<
"ORIGIN: " << *V <<
" ==> " << *Origin <<
"\n");
1872 OriginMap[
V] = Origin;
1876 Type *ShadowTy = getShadowTy(OrigTy);
1886 Constant *getCleanShadow(
Value *V) {
return getCleanShadow(
V->getType()); }
1891 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1893 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1895 getPoisonedShadow(AT->getElementType()));
1898 if (
StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1900 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1901 Vals.
push_back(getPoisonedShadow(
ST->getElementType(i)));
1909 Type *ShadowTy = getShadowTy(V);
1912 return getPoisonedShadow(ShadowTy);
1924 if (!PropagateShadow ||
I->getMetadata(LLVMContext::MD_nosanitize))
1925 return getCleanShadow(V);
1927 Value *Shadow = ShadowMap[
V];
1929 LLVM_DEBUG(
dbgs() <<
"No shadow: " << *V <<
"\n" << *(
I->getParent()));
1931 assert(Shadow &&
"No shadow for a value");
1935 if (
UndefValue *U = dyn_cast<UndefValue>(V)) {
1936 Value *
AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1937 : getCleanShadow(V);
1942 if (
Argument *
A = dyn_cast<Argument>(V)) {
1944 Value *&ShadowPtr = ShadowMap[
V];
1949 unsigned ArgOffset = 0;
1951 for (
auto &FArg :
F->args()) {
1952 if (!FArg.getType()->isSized()) {
1957 unsigned Size = FArg.hasByValAttr()
1958 ?
DL.getTypeAllocSize(FArg.getParamByValType())
1959 :
DL.getTypeAllocSize(FArg.getType());
1963 if (FArg.hasByValAttr()) {
1967 const Align ArgAlign =
DL.getValueOrABITypeAlignment(
1968 FArg.getParamAlign(), FArg.getParamByValType());
1969 Value *CpShadowPtr, *CpOriginPtr;
1970 std::tie(CpShadowPtr, CpOriginPtr) =
1971 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1973 if (!PropagateShadow || Overflow) {
1975 EntryIRB.CreateMemSet(
1979 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
1981 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign,
Base,
1986 if (MS.TrackOrigins) {
1988 getOriginPtrForArgument(EntryIRB, ArgOffset);
1992 EntryIRB.CreateMemCpy(
2001 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2002 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
2003 ShadowPtr = getCleanShadow(V);
2004 setOrigin(
A, getCleanOrigin());
2007 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2008 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg),
Base,
2010 if (MS.TrackOrigins) {
2012 getOriginPtrForArgument(EntryIRB, ArgOffset);
2013 setOrigin(
A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
2017 <<
" ARG: " << FArg <<
" ==> " << *ShadowPtr <<
"\n");
2023 assert(ShadowPtr &&
"Could not find shadow for an argument");
2027 return getCleanShadow(V);
2032 return getShadow(
I->getOperand(i));
2037 if (!MS.TrackOrigins)
2039 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
2040 return getCleanOrigin();
2041 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2042 "Unexpected value type in getOrigin()");
2044 if (
I->getMetadata(LLVMContext::MD_nosanitize))
2045 return getCleanOrigin();
2047 Value *Origin = OriginMap[
V];
2048 assert(Origin &&
"Missing origin");
2054 return getOrigin(
I->getOperand(i));
2067 LLVM_DEBUG(
dbgs() <<
"Skipping check of " << *Shadow <<
" before "
2068 << *OrigIns <<
"\n");
2073 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2074 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2075 "Can only insert checks for integer, vector, and aggregate shadow "
2078 InstrumentationList.push_back(
2079 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2088 Value *Shadow, *Origin;
2090 Shadow = getShadow(Val);
2093 Origin = getOrigin(Val);
2095 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2098 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
2100 insertShadowCheck(Shadow, Origin, OrigIns);
2105 case AtomicOrdering::NotAtomic:
2106 return AtomicOrdering::NotAtomic;
2107 case AtomicOrdering::Unordered:
2108 case AtomicOrdering::Monotonic:
2109 case AtomicOrdering::Release:
2110 return AtomicOrdering::Release;
2111 case AtomicOrdering::Acquire:
2112 case AtomicOrdering::AcquireRelease:
2113 return AtomicOrdering::AcquireRelease;
2114 case AtomicOrdering::SequentiallyConsistent:
2115 return AtomicOrdering::SequentiallyConsistent;
2121 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2122 uint32_t OrderingTable[NumOrderings] = {};
2124 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2125 OrderingTable[(
int)AtomicOrderingCABI::release] =
2126 (int)AtomicOrderingCABI::release;
2127 OrderingTable[(int)AtomicOrderingCABI::consume] =
2128 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2129 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2130 (
int)AtomicOrderingCABI::acq_rel;
2131 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2132 (
int)AtomicOrderingCABI::seq_cst;
2135 ArrayRef(OrderingTable, NumOrderings));
2140 case AtomicOrdering::NotAtomic:
2141 return AtomicOrdering::NotAtomic;
2142 case AtomicOrdering::Unordered:
2143 case AtomicOrdering::Monotonic:
2144 case AtomicOrdering::Acquire:
2145 return AtomicOrdering::Acquire;
2146 case AtomicOrdering::Release:
2147 case AtomicOrdering::AcquireRelease:
2148 return AtomicOrdering::AcquireRelease;
2149 case AtomicOrdering::SequentiallyConsistent:
2150 return AtomicOrdering::SequentiallyConsistent;
2156 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2157 uint32_t OrderingTable[NumOrderings] = {};
2159 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2160 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2161 OrderingTable[(int)AtomicOrderingCABI::consume] =
2162 (
int)AtomicOrderingCABI::acquire;
2163 OrderingTable[(int)AtomicOrderingCABI::release] =
2164 OrderingTable[(
int)AtomicOrderingCABI::acq_rel] =
2165 (int)AtomicOrderingCABI::acq_rel;
2166 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2167 (
int)AtomicOrderingCABI::seq_cst;
2170 ArrayRef(OrderingTable, NumOrderings));
2176 if (
I.getMetadata(LLVMContext::MD_nosanitize))
2179 if (isInPrologue(
I))
2189 assert(
I.getType()->isSized() &&
"Load type must have size");
2190 assert(!
I.getMetadata(LLVMContext::MD_nosanitize));
2191 NextNodeIRBuilder IRB(&
I);
2192 Type *ShadowTy = getShadowTy(&
I);
2194 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2195 const Align Alignment =
I.getAlign();
2196 if (PropagateShadow) {
2197 std::tie(ShadowPtr, OriginPtr) =
2198 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2202 setShadow(&
I, getCleanShadow(&
I));
2206 insertShadowCheck(
I.getPointerOperand(), &
I);
2211 if (MS.TrackOrigins) {
2212 if (PropagateShadow) {
2217 setOrigin(&
I, getCleanOrigin());
2227 StoreList.push_back(&
I);
2229 insertShadowCheck(
I.getPointerOperand(), &
I);
2233 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2237 Value *Val =
I.getOperand(1);
2238 Value *ShadowPtr = getShadowOriginPtr(
Addr, IRB, getShadowTy(Val),
Align(1),
2243 insertShadowCheck(
Addr, &
I);
2248 if (isa<AtomicCmpXchgInst>(
I))
2249 insertShadowCheck(Val, &
I);
2253 setShadow(&
I, getCleanShadow(&
I));
2254 setOrigin(&
I, getCleanOrigin());
2269 insertShadowCheck(
I.getOperand(1), &
I);
2273 setOrigin(&
I, getOrigin(&
I, 0));
2277 insertShadowCheck(
I.getOperand(2), &
I);
2279 auto *Shadow0 = getShadow(&
I, 0);
2280 auto *Shadow1 = getShadow(&
I, 1);
2283 setOriginForNaryOp(
I);
2288 auto *Shadow0 = getShadow(&
I, 0);
2289 auto *Shadow1 = getShadow(&
I, 1);
2292 setOriginForNaryOp(
I);
2298 setShadow(&
I, IRB.
CreateSExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2299 setOrigin(&
I, getOrigin(&
I, 0));
2304 setShadow(&
I, IRB.
CreateZExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2305 setOrigin(&
I, getOrigin(&
I, 0));
2310 setShadow(&
I, IRB.
CreateTrunc(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2311 setOrigin(&
I, getOrigin(&
I, 0));
2318 if (
auto *CI = dyn_cast<CallInst>(
I.getOperand(0)))
2319 if (CI->isMustTailCall())
2323 setOrigin(&
I, getOrigin(&
I, 0));
2329 "_msprop_ptrtoint"));
2330 setOrigin(&
I, getOrigin(&
I, 0));
2336 "_msprop_inttoptr"));
2337 setOrigin(&
I, getOrigin(&
I, 0));
2340 void visitFPToSIInst(
CastInst &
I) { handleShadowOr(
I); }
2341 void visitFPToUIInst(
CastInst &
I) { handleShadowOr(
I); }
2342 void visitSIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2343 void visitUIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2344 void visitFPExtInst(
CastInst &
I) { handleShadowOr(
I); }
2345 void visitFPTruncInst(
CastInst &
I) { handleShadowOr(
I); }
2360 Value *S2 = getShadow(&
I, 1);
2361 Value *V1 =
I.getOperand(0);
2370 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2371 setOriginForNaryOp(
I);
2382 Value *S2 = getShadow(&
I, 1);
2392 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2393 setOriginForNaryOp(
I);
2411 template <
bool CombineShadow>
class Combiner {
2412 Value *Shadow =
nullptr;
2413 Value *Origin =
nullptr;
2415 MemorySanitizerVisitor *MSV;
2419 : IRB(IRB), MSV(MSV) {}
2423 if (CombineShadow) {
2428 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2429 Shadow = IRB.
CreateOr(Shadow, OpShadow,
"_msprop");
2433 if (MSV->MS.TrackOrigins) {
2438 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2440 if (!ConstOrigin || !ConstOrigin->
isNullValue()) {
2441 Value *
Cond = MSV->convertToBool(OpShadow, IRB);
2451 Value *OpShadow = MSV->getShadow(V);
2452 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) :
nullptr;
2453 return Add(OpShadow, OpOrigin);
2459 if (CombineShadow) {
2461 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(
I));
2462 MSV->setShadow(
I, Shadow);
2464 if (MSV->MS.TrackOrigins) {
2466 MSV->setOrigin(
I, Origin);
2476 if (!MS.TrackOrigins)
2479 OriginCombiner
OC(
this, IRB);
2480 for (
Use &
Op :
I.operands())
2485 size_t VectorOrPrimitiveTypeSizeInBits(
Type *Ty) {
2487 "Vector of pointers is not a valid shadow type");
2488 return Ty->
isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2497 Type *srcTy =
V->getType();
2498 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2499 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2500 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2506 cast<VectorType>(dstTy)->getElementCount() ==
2507 cast<VectorType>(srcTy)->getElementCount())
2518 Type *ShadowTy = getShadowTy(V);
2519 if (
V->getType() == ShadowTy)
2521 if (
V->getType()->isPtrOrPtrVectorTy())
2530 ShadowAndOriginCombiner
SC(
this, IRB);
2531 for (
Use &
Op :
I.operands())
2551 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
2552 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2553 Type *EltTy = VTy->getElementType();
2555 for (
unsigned Idx = 0;
Idx < NumElements; ++
Idx) {
2558 const APInt &
V = Elt->getValue();
2560 Elements.push_back(ConstantInt::get(EltTy, V2));
2562 Elements.push_back(ConstantInt::get(EltTy, 1));
2567 if (
ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2568 const APInt &
V = Elt->getValue();
2570 ShadowMul = ConstantInt::get(Ty, V2);
2572 ShadowMul = ConstantInt::get(Ty, 1);
2578 IRB.
CreateMul(getShadow(OtherArg), ShadowMul,
"msprop_mul_cst"));
2579 setOrigin(&
I, getOrigin(OtherArg));
2583 Constant *constOp0 = dyn_cast<Constant>(
I.getOperand(0));
2584 Constant *constOp1 = dyn_cast<Constant>(
I.getOperand(1));
2585 if (constOp0 && !constOp1)
2586 handleMulByConstant(
I, constOp0,
I.getOperand(1));
2587 else if (constOp1 && !constOp0)
2588 handleMulByConstant(
I, constOp1,
I.getOperand(0));
2603 insertShadowCheck(
I.getOperand(1), &
I);
2604 setShadow(&
I, getShadow(&
I, 0));
2605 setOrigin(&
I, getOrigin(&
I, 0));
2622 void handleEqualityComparison(
ICmpInst &
I) {
2626 Value *Sa = getShadow(
A);
2627 Value *Sb = getShadow(
B);
2653 setOriginForNaryOp(
I);
2695 void handleRelationalComparisonExact(
ICmpInst &
I) {
2699 Value *Sa = getShadow(
A);
2700 Value *Sb = getShadow(
B);
2711 bool IsSigned =
I.isSigned();
2713 getLowestPossibleValue(IRB,
A, Sa, IsSigned),
2714 getHighestPossibleValue(IRB,
B, Sb, IsSigned));
2716 getHighestPossibleValue(IRB,
A, Sa, IsSigned),
2717 getLowestPossibleValue(IRB,
B, Sb, IsSigned));
2720 setOriginForNaryOp(
I);
2727 void handleSignedRelationalComparison(
ICmpInst &
I) {
2731 if ((constOp = dyn_cast<Constant>(
I.getOperand(1)))) {
2732 op =
I.getOperand(0);
2733 pre =
I.getPredicate();
2734 }
else if ((constOp = dyn_cast<Constant>(
I.getOperand(0)))) {
2735 op =
I.getOperand(1);
2736 pre =
I.getSwappedPredicate();
2749 setShadow(&
I, Shadow);
2750 setOrigin(&
I, getOrigin(
op));
2761 if (
I.isEquality()) {
2762 handleEqualityComparison(
I);
2768 handleRelationalComparisonExact(
I);
2772 handleSignedRelationalComparison(
I);
2777 if ((isa<Constant>(
I.getOperand(0)) || isa<Constant>(
I.getOperand(1)))) {
2778 handleRelationalComparisonExact(
I);
2785 void visitFCmpInst(
FCmpInst &
I) { handleShadowOr(
I); }
2792 Value *S2 = getShadow(&
I, 1);
2797 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2798 setOriginForNaryOp(
I);
2809 Value *S0 = getShadow(&
I, 0);
2811 Value *S2 = getShadow(&
I, 2);
2816 I.getModule(),
I.getIntrinsicID(), S2Conv->
getType());
2818 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2819 setOriginForNaryOp(
I);
2833 getShadow(
I.getArgOperand(1));
2836 {I.getArgOperand(0), I.getArgOperand(1),
2837 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2838 I.eraseFromParent();
2856 getShadow(
I.getArgOperand(1));
2859 {I.getArgOperand(0), I.getArgOperand(1),
2860 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2861 I.eraseFromParent();
2869 {I.getArgOperand(0),
2870 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2871 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2872 I.eraseFromParent();
2875 void visitVAStartInst(
VAStartInst &
I) { VAHelper->visitVAStartInst(
I); }
2877 void visitVACopyInst(
VACopyInst &
I) { VAHelper->visitVACopyInst(
I); }
2886 Value *Shadow = getShadow(&
I, 1);
2887 Value *ShadowPtr, *OriginPtr;
2891 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2896 insertShadowCheck(
Addr, &
I);
2899 if (MS.TrackOrigins)
2912 Type *ShadowTy = getShadowTy(&
I);
2913 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2914 if (PropagateShadow) {
2918 std::tie(ShadowPtr, OriginPtr) =
2919 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2923 setShadow(&
I, getCleanShadow(&
I));
2927 insertShadowCheck(
Addr, &
I);
2929 if (MS.TrackOrigins) {
2930 if (PropagateShadow)
2931 setOrigin(&
I, IRB.
CreateLoad(MS.OriginTy, OriginPtr));
2933 setOrigin(&
I, getCleanOrigin());
2946 if (!(
RetTy->isIntOrIntVectorTy() ||
RetTy->isFPOrFPVectorTy() ||
2947 RetTy->isX86_MMXTy()))
2950 unsigned NumArgOperands =
I.arg_size();
2951 for (
unsigned i = 0; i < NumArgOperands; ++i) {
2952 Type *Ty =
I.getArgOperand(i)->getType();
2958 ShadowAndOriginCombiner
SC(
this, IRB);
2959 for (
unsigned i = 0; i < NumArgOperands; ++i)
2960 SC.Add(
I.getArgOperand(i));
2977 unsigned NumArgOperands =
I.arg_size();
2978 if (NumArgOperands == 0)
2981 if (NumArgOperands == 2 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
2982 I.getArgOperand(1)->getType()->isVectorTy() &&
2983 I.getType()->isVoidTy() && !
I.onlyReadsMemory()) {
2985 return handleVectorStoreIntrinsic(
I);
2988 if (NumArgOperands == 1 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
2989 I.getType()->isVectorTy() &&
I.onlyReadsMemory()) {
2991 return handleVectorLoadIntrinsic(
I);
2994 if (
I.doesNotAccessMemory())
2995 if (maybeHandleSimpleNomemIntrinsic(
I))
3003 setShadow(&
I, getShadow(&
I, 0));
3004 setOrigin(&
I, getOrigin(&
I, 0));
3012 InstrumentLifetimeStart =
false;
3013 LifetimeStartList.push_back(std::make_pair(&
I, AI));
3019 Type *OpType =
Op->getType();
3021 F.getParent(), Intrinsic::bswap,
ArrayRef(&OpType, 1));
3023 setOrigin(&
I, getOrigin(
Op));
3028 Value *Src =
I.getArgOperand(0);
3034 Constant *IsZeroPoison = cast<Constant>(
I.getOperand(1));
3037 BoolShadow = IRB.
CreateOr(BoolShadow, BoolZeroPoison,
"_mscz_bs");
3040 Value *OutputShadow =
3041 IRB.
CreateSExt(BoolShadow, getShadowTy(Src),
"_mscz_os");
3043 setShadow(&
I, OutputShadow);
3044 setOriginForNaryOp(
I);
3062 void handleVectorConvertIntrinsic(
IntrinsicInst &
I,
int NumUsedElements,
3063 bool HasRoundingMode =
false) {
3065 Value *CopyOp, *ConvertOp;
3067 assert((!HasRoundingMode ||
3068 isa<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))) &&
3069 "Invalid rounding mode");
3071 switch (
I.arg_size() - HasRoundingMode) {
3073 CopyOp =
I.getArgOperand(0);
3074 ConvertOp =
I.getArgOperand(1);
3077 ConvertOp =
I.getArgOperand(0);
3091 Value *ConvertShadow = getShadow(ConvertOp);
3092 Value *AggShadow =
nullptr;
3095 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), 0));
3096 for (
int i = 1; i < NumUsedElements; ++i) {
3098 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), i));
3099 AggShadow = IRB.
CreateOr(AggShadow, MoreShadow);
3102 AggShadow = ConvertShadow;
3105 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &
I);
3112 Value *ResultShadow = getShadow(CopyOp);
3113 Type *EltTy = cast<VectorType>(ResultShadow->
getType())->getElementType();
3114 for (
int i = 0; i < NumUsedElements; ++i) {
3116 ResultShadow, ConstantInt::getNullValue(EltTy),
3119 setShadow(&
I, ResultShadow);
3120 setOrigin(&
I, getOrigin(CopyOp));
3122 setShadow(&
I, getCleanShadow(&
I));
3123 setOrigin(&
I, getCleanOrigin());
3131 S = CreateShadowCast(IRB, S, IRB.
getInt64Ty(),
true);
3134 return CreateShadowCast(IRB, S2,
T,
true);
3142 return CreateShadowCast(IRB, S2,
T,
true);
3159 void handleVectorShiftIntrinsic(
IntrinsicInst &
I,
bool Variable) {
3165 Value *S2 = getShadow(&
I, 1);
3166 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3167 : Lower64ShadowExtend(IRB, S2, getShadowTy(&
I));
3168 Value *V1 =
I.getOperand(0);
3171 {IRB.CreateBitCast(S1, V1->getType()), V2});
3173 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
3174 setOriginForNaryOp(
I);
3178 Type *getMMXVectorTy(
unsigned EltSizeInBits) {
3179 const unsigned X86_MMXSizeInBits = 64;
3180 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3181 "Illegal MMX vector element size");
3183 X86_MMXSizeInBits / EltSizeInBits);
3190 case Intrinsic::x86_sse2_packsswb_128:
3191 case Intrinsic::x86_sse2_packuswb_128:
3192 return Intrinsic::x86_sse2_packsswb_128;
3194 case Intrinsic::x86_sse2_packssdw_128:
3195 case Intrinsic::x86_sse41_packusdw:
3196 return Intrinsic::x86_sse2_packssdw_128;
3198 case Intrinsic::x86_avx2_packsswb:
3199 case Intrinsic::x86_avx2_packuswb:
3200 return Intrinsic::x86_avx2_packsswb;
3202 case Intrinsic::x86_avx2_packssdw:
3203 case Intrinsic::x86_avx2_packusdw:
3204 return Intrinsic::x86_avx2_packssdw;
3206 case Intrinsic::x86_mmx_packsswb:
3207 case Intrinsic::x86_mmx_packuswb:
3208 return Intrinsic::x86_mmx_packsswb;
3210 case Intrinsic::x86_mmx_packssdw:
3211 return Intrinsic::x86_mmx_packssdw;
3224 void handleVectorPackIntrinsic(
IntrinsicInst &
I,
unsigned EltSizeInBits = 0) {
3226 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3229 Value *S2 = getShadow(&
I, 1);
3230 assert(isX86_MMX ||
S1->getType()->isVectorTy());
3235 Type *
T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) :
S1->
getType();
3251 F.getParent(), getSignedPackIntrinsic(
I.getIntrinsicID()));
3254 IRB.
CreateCall(ShadowFn, {S1_ext, S2_ext},
"_msprop_vector_pack");
3258 setOriginForNaryOp(
I);
3263 const unsigned SignificantBitsPerResultElement = 16;
3264 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3266 unsigned ZeroBitsPerResultElement =
3270 auto *Shadow0 = getShadow(&
I, 0);
3271 auto *Shadow1 = getShadow(&
I, 1);
3276 S = IRB.
CreateLShr(S, ZeroBitsPerResultElement);
3279 setOriginForNaryOp(
I);
3284 unsigned EltSizeInBits = 0) {
3285 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3286 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) :
I.
getType();
3288 auto *Shadow0 = getShadow(&
I, 0);
3289 auto *Shadow1 = getShadow(&
I, 1);
3296 setOriginForNaryOp(
I);
3304 Type *ResTy = getShadowTy(&
I);
3305 auto *Shadow0 = getShadow(&
I, 0);
3306 auto *Shadow1 = getShadow(&
I, 1);
3311 setOriginForNaryOp(
I);
3319 auto *Shadow0 = getShadow(&
I, 0);
3320 auto *Shadow1 = getShadow(&
I, 1);
3322 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&
I));
3324 setOriginForNaryOp(
I);
3333 setOrigin(&
I, getOrigin(&
I, 0));
3341 Value *OperandShadow = getShadow(&
I, 0);
3343 Value *OperandUnsetOrPoison = IRB.
CreateOr(OperandUnsetBits, OperandShadow);
3351 setOrigin(&
I, getOrigin(&
I, 0));
3359 Value *OperandShadow = getShadow(&
I, 0);
3360 Value *OperandSetOrPoison = IRB.
CreateOr(
I.getOperand(0), OperandShadow);
3368 setOrigin(&
I, getOrigin(&
I, 0));
3376 getShadowOriginPtr(
Addr, IRB, Ty,
Align(1),
true).first;
3381 insertShadowCheck(
Addr, &
I);
3392 Value *ShadowPtr, *OriginPtr;
3393 std::tie(ShadowPtr, OriginPtr) =
3394 getShadowOriginPtr(
Addr, IRB, Ty, Alignment,
false);
3397 insertShadowCheck(
Addr, &
I);
3400 Value *Origin = MS.TrackOrigins ? IRB.
CreateLoad(MS.OriginTy, OriginPtr)
3402 insertShadowCheck(Shadow, Origin, &
I);
3409 Value *PassThru =
I.getArgOperand(2);
3412 insertShadowCheck(
Ptr, &
I);
3413 insertShadowCheck(Mask, &
I);
3416 if (!PropagateShadow) {
3417 setShadow(&
I, getCleanShadow(&
I));
3418 setOrigin(&
I, getCleanOrigin());
3422 Type *ShadowTy = getShadowTy(&
I);
3423 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3424 auto [ShadowPtr, OriginPtr] =
3425 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
false);
3428 ShadowTy, ShadowPtr, Mask, getShadow(PassThru),
"_msmaskedexpload");
3430 setShadow(&
I, Shadow);
3433 setOrigin(&
I, getCleanOrigin());
3438 Value *Values =
I.getArgOperand(0);
3443 insertShadowCheck(
Ptr, &
I);
3444 insertShadowCheck(Mask, &
I);
3447 Value *Shadow = getShadow(Values);
3448 Type *ElementShadowTy =
3449 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3450 auto [ShadowPtr, OriginPtrs] =
3451 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
true);
3460 Value *Ptrs =
I.getArgOperand(0);
3461 const Align Alignment(
3462 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3464 Value *PassThru =
I.getArgOperand(3);
3466 Type *PtrsShadowTy = getShadowTy(Ptrs);
3468 insertShadowCheck(Mask, &
I);
3472 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3475 if (!PropagateShadow) {
3476 setShadow(&
I, getCleanShadow(&
I));
3477 setOrigin(&
I, getCleanOrigin());
3481 Type *ShadowTy = getShadowTy(&
I);
3482 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3483 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3484 Ptrs, IRB, ElementShadowTy, Alignment,
false);
3488 getShadow(PassThru),
"_msmaskedgather");
3490 setShadow(&
I, Shadow);
3493 setOrigin(&
I, getCleanOrigin());
3498 Value *Values =
I.getArgOperand(0);
3499 Value *Ptrs =
I.getArgOperand(1);
3500 const Align Alignment(
3501 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3504 Type *PtrsShadowTy = getShadowTy(Ptrs);
3506 insertShadowCheck(Mask, &
I);
3510 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3513 Value *Shadow = getShadow(Values);
3514 Type *ElementShadowTy =
3515 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3516 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3517 Ptrs, IRB, ElementShadowTy, Alignment,
true);
3526 Value *
V =
I.getArgOperand(0);
3528 const Align Alignment(
3529 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3531 Value *Shadow = getShadow(V);
3534 insertShadowCheck(
Ptr, &
I);
3535 insertShadowCheck(Mask, &
I);
3540 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3541 Ptr, IRB, Shadow->
getType(), Alignment,
true);
3545 if (!MS.TrackOrigins)
3548 auto &
DL =
F.getParent()->getDataLayout();
3549 paintOrigin(IRB, getOrigin(V), OriginPtr,
3557 const Align Alignment(
3558 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3560 Value *PassThru =
I.getArgOperand(3);
3563 insertShadowCheck(
Ptr, &
I);
3564 insertShadowCheck(Mask, &
I);
3567 if (!PropagateShadow) {
3568 setShadow(&
I, getCleanShadow(&
I));
3569 setOrigin(&
I, getCleanOrigin());
3573 Type *ShadowTy = getShadowTy(&
I);
3574 Value *ShadowPtr, *OriginPtr;
3575 std::tie(ShadowPtr, OriginPtr) =
3576 getShadowOriginPtr(
Ptr, IRB, ShadowTy, Alignment,
false);
3578 getShadow(PassThru),
"_msmaskedld"));
3580 if (!MS.TrackOrigins)
3587 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB,
"_mscmp");
3592 setOrigin(&
I, Origin);
3602 Type *ShadowTy = getShadowTy(&
I);
3605 Value *SMask = getShadow(&
I, 1);
3610 {getShadow(&I, 0), I.getOperand(1)});
3613 setOriginForNaryOp(
I);
3618 for (
unsigned X = OddElements ? 1 : 0;
X < Width;
X += 2) {
3635 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3636 assert(isa<ConstantInt>(
I.getArgOperand(2)) &&
3637 "pclmul 3rd operand must be a constant");
3638 unsigned Imm = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3640 getPclmulMask(Width, Imm & 0x01));
3642 getPclmulMask(Width, Imm & 0x10));
3643 ShadowAndOriginCombiner SOC(
this, IRB);
3644 SOC.Add(Shuf0, getOrigin(&
I, 0));
3645 SOC.Add(Shuf1, getOrigin(&
I, 1));
3653 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3655 Value *Second = getShadow(&
I, 1);
3658 Mask.push_back(Width);
3659 for (
unsigned i = 1; i < Width; i++)
3663 setShadow(&
I, Shadow);
3664 setOriginForNaryOp(
I);
3669 Value *Shadow0 = getShadow(&
I, 0);
3670 Value *Shadow1 = getShadow(&
I, 1);
3676 setShadow(&
I, Shadow);
3677 setOriginForNaryOp(
I);
3683 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3685 Value *Second = getShadow(&
I, 1);
3689 Mask.push_back(Width);
3690 for (
unsigned i = 1; i < Width; i++)
3694 setShadow(&
I, Shadow);
3695 setOriginForNaryOp(
I);
3702 assert(
I.getType()->isIntOrIntVectorTy());
3703 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3707 setShadow(&
I, getShadow(&
I, 0));
3708 setOrigin(&
I, getOrigin(&
I, 0));
3713 Value *Shadow = getShadow(&
I, 0);
3714 setShadow(&
I, IRB.
CreateICmpNE(Shadow, getCleanShadow(Shadow)));
3715 setOrigin(&
I, getOrigin(&
I, 0));
3720 Value *Shadow0 = getShadow(&
I, 0);
3721 Value *Shadow1 = getShadow(&
I, 1);
3724 IRB.
CreateICmpNE(ShadowElt0, getCleanShadow(ShadowElt0));
3730 setShadow(&
I, Shadow);
3731 setOriginForNaryOp(
I);
3735 switch (
I.getIntrinsicID()) {
3736 case Intrinsic::uadd_with_overflow:
3737 case Intrinsic::sadd_with_overflow:
3738 case Intrinsic::usub_with_overflow:
3739 case Intrinsic::ssub_with_overflow:
3740 case Intrinsic::umul_with_overflow:
3741 case Intrinsic::smul_with_overflow:
3742 handleArithmeticWithOverflow(
I);
3744 case Intrinsic::abs:
3745 handleAbsIntrinsic(
I);
3747 case Intrinsic::is_fpclass:
3750 case Intrinsic::lifetime_start:
3751 handleLifetimeStart(
I);
3753 case Intrinsic::launder_invariant_group:
3754 case Intrinsic::strip_invariant_group:
3755 handleInvariantGroup(
I);
3757 case Intrinsic::bswap:
3760 case Intrinsic::ctlz:
3761 case Intrinsic::cttz:
3762 handleCountZeroes(
I);
3764 case Intrinsic::masked_compressstore:
3765 handleMaskedCompressStore(
I);
3767 case Intrinsic::masked_expandload:
3768 handleMaskedExpandLoad(
I);
3770 case Intrinsic::masked_gather:
3771 handleMaskedGather(
I);
3773 case Intrinsic::masked_scatter:
3774 handleMaskedScatter(
I);
3776 case Intrinsic::masked_store:
3777 handleMaskedStore(
I);
3779 case Intrinsic::masked_load:
3780 handleMaskedLoad(
I);
3782 case Intrinsic::vector_reduce_and:
3783 handleVectorReduceAndIntrinsic(
I);
3785 case Intrinsic::vector_reduce_or:
3786 handleVectorReduceOrIntrinsic(
I);
3788 case Intrinsic::vector_reduce_add:
3789 case Intrinsic::vector_reduce_xor:
3790 case Intrinsic::vector_reduce_mul:
3791 handleVectorReduceIntrinsic(
I);
3793 case Intrinsic::x86_sse_stmxcsr:
3796 case Intrinsic::x86_sse_ldmxcsr:
3799 case Intrinsic::x86_avx512_vcvtsd2usi64:
3800 case Intrinsic::x86_avx512_vcvtsd2usi32:
3801 case Intrinsic::x86_avx512_vcvtss2usi64:
3802 case Intrinsic::x86_avx512_vcvtss2usi32:
3803 case Intrinsic::x86_avx512_cvttss2usi64:
3804 case Intrinsic::x86_avx512_cvttss2usi:
3805 case Intrinsic::x86_avx512_cvttsd2usi64:
3806 case Intrinsic::x86_avx512_cvttsd2usi:
3807 case Intrinsic::x86_avx512_cvtusi2ss:
3808 case Intrinsic::x86_avx512_cvtusi642sd:
3809 case Intrinsic::x86_avx512_cvtusi642ss:
3810 handleVectorConvertIntrinsic(
I, 1,
true);
3812 case Intrinsic::x86_sse2_cvtsd2si64:
3813 case Intrinsic::x86_sse2_cvtsd2si:
3814 case Intrinsic::x86_sse2_cvtsd2ss:
3815 case Intrinsic::x86_sse2_cvttsd2si64:
3816 case Intrinsic::x86_sse2_cvttsd2si:
3817 case Intrinsic::x86_sse_cvtss2si64:
3818 case Intrinsic::x86_sse_cvtss2si:
3819 case Intrinsic::x86_sse_cvttss2si64:
3820 case Intrinsic::x86_sse_cvttss2si:
3821 handleVectorConvertIntrinsic(
I, 1);
3823 case Intrinsic::x86_sse_cvtps2pi:
3824 case Intrinsic::x86_sse_cvttps2pi:
3825 handleVectorConvertIntrinsic(
I, 2);
3828 case Intrinsic::x86_avx512_psll_w_512:
3829 case Intrinsic::x86_avx512_psll_d_512:
3830 case Intrinsic::x86_avx512_psll_q_512:
3831 case Intrinsic::x86_avx512_pslli_w_512:
3832 case Intrinsic::x86_avx512_pslli_d_512:
3833 case Intrinsic::x86_avx512_pslli_q_512:
3834 case Intrinsic::x86_avx512_psrl_w_512:
3835 case Intrinsic::x86_avx512_psrl_d_512:
3836 case Intrinsic::x86_avx512_psrl_q_512:
3837 case Intrinsic::x86_avx512_psra_w_512:
3838 case Intrinsic::x86_avx512_psra_d_512:
3839 case Intrinsic::x86_avx512_psra_q_512:
3840 case Intrinsic::x86_avx512_psrli_w_512:
3841 case Intrinsic::x86_avx512_psrli_d_512:
3842 case Intrinsic::x86_avx512_psrli_q_512:
3843 case Intrinsic::x86_avx512_psrai_w_512:
3844 case Intrinsic::x86_avx512_psrai_d_512:
3845 case Intrinsic::x86_avx512_psrai_q_512:
3846 case Intrinsic::x86_avx512_psra_q_256:
3847 case Intrinsic::x86_avx512_psra_q_128:
3848 case Intrinsic::x86_avx512_psrai_q_256:
3849 case Intrinsic::x86_avx512_psrai_q_128:
3850 case Intrinsic::x86_avx2_psll_w:
3851 case Intrinsic::x86_avx2_psll_d:
3852 case Intrinsic::x86_avx2_psll_q:
3853 case Intrinsic::x86_avx2_pslli_w:
3854 case Intrinsic::x86_avx2_pslli_d:
3855 case Intrinsic::x86_avx2_pslli_q:
3856 case Intrinsic::x86_avx2_psrl_w:
3857 case Intrinsic::x86_avx2_psrl_d:
3858 case Intrinsic::x86_avx2_psrl_q:
3859 case Intrinsic::x86_avx2_psra_w:
3860 case Intrinsic::x86_avx2_psra_d:
3861 case Intrinsic::x86_avx2_psrli_w:
3862 case Intrinsic::x86_avx2_psrli_d:
3863 case Intrinsic::x86_avx2_psrli_q:
3864 case Intrinsic::x86_avx2_psrai_w:
3865 case Intrinsic::x86_avx2_psrai_d:
3866 case Intrinsic::x86_sse2_psll_w:
3867 case Intrinsic::x86_sse2_psll_d:
3868 case Intrinsic::x86_sse2_psll_q:
3869 case Intrinsic::x86_sse2_pslli_w:
3870 case Intrinsic::x86_sse2_pslli_d:
3871 case Intrinsic::x86_sse2_pslli_q:
3872 case Intrinsic::x86_sse2_psrl_w:
3873 case Intrinsic::x86_sse2_psrl_d:
3874 case Intrinsic::x86_sse2_psrl_q:
3875 case Intrinsic::x86_sse2_psra_w:
3876 case Intrinsic::x86_sse2_psra_d:
3877 case Intrinsic::x86_sse2_psrli_w:
3878 case Intrinsic::x86_sse2_psrli_d:
3879 case Intrinsic::x86_sse2_psrli_q:
3880 case Intrinsic::x86_sse2_psrai_w:
3881 case Intrinsic::x86_sse2_psrai_d:
3882 case Intrinsic::x86_mmx_psll_w:
3883 case Intrinsic::x86_mmx_psll_d:
3884 case Intrinsic::x86_mmx_psll_q:
3885 case Intrinsic::x86_mmx_pslli_w:
3886 case Intrinsic::x86_mmx_pslli_d:
3887 case Intrinsic::x86_mmx_pslli_q:
3888 case Intrinsic::x86_mmx_psrl_w:
3889 case Intrinsic::x86_mmx_psrl_d:
3890 case Intrinsic::x86_mmx_psrl_q:
3891 case Intrinsic::x86_mmx_psra_w:
3892 case Intrinsic::x86_mmx_psra_d:
3893 case Intrinsic::x86_mmx_psrli_w:
3894 case Intrinsic::x86_mmx_psrli_d:
3895 case Intrinsic::x86_mmx_psrli_q:
3896 case Intrinsic::x86_mmx_psrai_w:
3897 case Intrinsic::x86_mmx_psrai_d:
3898 handleVectorShiftIntrinsic(
I,
false);
3900 case Intrinsic::x86_avx2_psllv_d:
3901 case Intrinsic::x86_avx2_psllv_d_256:
3902 case Intrinsic::x86_avx512_psllv_d_512:
3903 case Intrinsic::x86_avx2_psllv_q:
3904 case Intrinsic::x86_avx2_psllv_q_256:
3905 case Intrinsic::x86_avx512_psllv_q_512:
3906 case Intrinsic::x86_avx2_psrlv_d:
3907 case Intrinsic::x86_avx2_psrlv_d_256:
3908 case Intrinsic::x86_avx512_psrlv_d_512:
3909 case Intrinsic::x86_avx2_psrlv_q:
3910 case Intrinsic::x86_avx2_psrlv_q_256:
3911 case Intrinsic::x86_avx512_psrlv_q_512:
3912 case Intrinsic::x86_avx2_psrav_d:
3913 case Intrinsic::x86_avx2_psrav_d_256:
3914 case Intrinsic::x86_avx512_psrav_d_512:
3915 case Intrinsic::x86_avx512_psrav_q_128:
3916 case Intrinsic::x86_avx512_psrav_q_256:
3917 case Intrinsic::x86_avx512_psrav_q_512:
3918 handleVectorShiftIntrinsic(
I,
true);
3921 case Intrinsic::x86_sse2_packsswb_128:
3922 case Intrinsic::x86_sse2_packssdw_128:
3923 case Intrinsic::x86_sse2_packuswb_128:
3924 case Intrinsic::x86_sse41_packusdw:
3925 case Intrinsic::x86_avx2_packsswb:
3926 case Intrinsic::x86_avx2_packssdw:
3927 case Intrinsic::x86_avx2_packuswb:
3928 case Intrinsic::x86_avx2_packusdw:
3929 handleVectorPackIntrinsic(
I);
3932 case Intrinsic::x86_mmx_packsswb:
3933 case Intrinsic::x86_mmx_packuswb:
3934 handleVectorPackIntrinsic(
I, 16);
3937 case Intrinsic::x86_mmx_packssdw:
3938 handleVectorPackIntrinsic(
I, 32);
3941 case Intrinsic::x86_mmx_psad_bw:
3942 case Intrinsic::x86_sse2_psad_bw:
3943 case Intrinsic::x86_avx2_psad_bw:
3944 handleVectorSadIntrinsic(
I);
3947 case Intrinsic::x86_sse2_pmadd_wd:
3948 case Intrinsic::x86_avx2_pmadd_wd:
3949 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3950 case Intrinsic::x86_avx2_pmadd_ub_sw:
3951 handleVectorPmaddIntrinsic(
I);
3954 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3955 handleVectorPmaddIntrinsic(
I, 8);
3958 case Intrinsic::x86_mmx_pmadd_wd:
3959 handleVectorPmaddIntrinsic(
I, 16);
3962 case Intrinsic::x86_sse_cmp_ss:
3963 case Intrinsic::x86_sse2_cmp_sd:
3964 case Intrinsic::x86_sse_comieq_ss:
3965 case Intrinsic::x86_sse_comilt_ss:
3966 case Intrinsic::x86_sse_comile_ss:
3967 case Intrinsic::x86_sse_comigt_ss:
3968 case Intrinsic::x86_sse_comige_ss:
3969 case Intrinsic::x86_sse_comineq_ss:
3970 case Intrinsic::x86_sse_ucomieq_ss:
3971 case Intrinsic::x86_sse_ucomilt_ss:
3972 case Intrinsic::x86_sse_ucomile_ss:
3973 case Intrinsic::x86_sse_ucomigt_ss:
3974 case Intrinsic::x86_sse_ucomige_ss:
3975 case Intrinsic::x86_sse_ucomineq_ss:
3976 case Intrinsic::x86_sse2_comieq_sd:
3977 case Intrinsic::x86_sse2_comilt_sd:
3978 case Intrinsic::x86_sse2_comile_sd:
3979 case Intrinsic::x86_sse2_comigt_sd:
3980 case Intrinsic::x86_sse2_comige_sd:
3981 case Intrinsic::x86_sse2_comineq_sd:
3982 case Intrinsic::x86_sse2_ucomieq_sd:
3983 case Intrinsic::x86_sse2_ucomilt_sd:
3984 case Intrinsic::x86_sse2_ucomile_sd:
3985 case Intrinsic::x86_sse2_ucomigt_sd:
3986 case Intrinsic::x86_sse2_ucomige_sd:
3987 case Intrinsic::x86_sse2_ucomineq_sd:
3988 handleVectorCompareScalarIntrinsic(
I);
3991 case Intrinsic::x86_avx_cmp_pd_256:
3992 case Intrinsic::x86_avx_cmp_ps_256:
3993 case Intrinsic::x86_sse2_cmp_pd:
3994 case Intrinsic::x86_sse_cmp_ps:
3995 handleVectorComparePackedIntrinsic(
I);
3998 case Intrinsic::x86_bmi_bextr_32:
3999 case Intrinsic::x86_bmi_bextr_64:
4000 case Intrinsic::x86_bmi_bzhi_32:
4001 case Intrinsic::x86_bmi_bzhi_64:
4002 case Intrinsic::x86_bmi_pdep_32:
4003 case Intrinsic::x86_bmi_pdep_64:
4004 case Intrinsic::x86_bmi_pext_32:
4005 case Intrinsic::x86_bmi_pext_64:
4006 handleBmiIntrinsic(
I);
4009 case Intrinsic::x86_pclmulqdq:
4010 case Intrinsic::x86_pclmulqdq_256:
4011 case Intrinsic::x86_pclmulqdq_512:
4012 handlePclmulIntrinsic(
I);
4015 case Intrinsic::x86_sse41_round_sd:
4016 case Intrinsic::x86_sse41_round_ss:
4017 handleUnarySdSsIntrinsic(
I);
4019 case Intrinsic::x86_sse2_max_sd:
4020 case Intrinsic::x86_sse_max_ss:
4021 case Intrinsic::x86_sse2_min_sd:
4022 case Intrinsic::x86_sse_min_ss:
4023 handleBinarySdSsIntrinsic(
I);
4026 case Intrinsic::x86_avx_vtestc_pd:
4027 case Intrinsic::x86_avx_vtestc_pd_256:
4028 case Intrinsic::x86_avx_vtestc_ps:
4029 case Intrinsic::x86_avx_vtestc_ps_256:
4030 case Intrinsic::x86_avx_vtestnzc_pd:
4031 case Intrinsic::x86_avx_vtestnzc_pd_256:
4032 case Intrinsic::x86_avx_vtestnzc_ps:
4033 case Intrinsic::x86_avx_vtestnzc_ps_256:
4034 case Intrinsic::x86_avx_vtestz_pd:
4035 case Intrinsic::x86_avx_vtestz_pd_256:
4036 case Intrinsic::x86_avx_vtestz_ps:
4037 case Intrinsic::x86_avx_vtestz_ps_256:
4038 case Intrinsic::x86_avx_ptestc_256:
4039 case Intrinsic::x86_avx_ptestnzc_256:
4040 case Intrinsic::x86_avx_ptestz_256:
4041 case Intrinsic::x86_sse41_ptestc:
4042 case Intrinsic::x86_sse41_ptestnzc:
4043 case Intrinsic::x86_sse41_ptestz:
4044 handleVtestIntrinsic(
I);
4047 case Intrinsic::fshl:
4048 case Intrinsic::fshr:
4049 handleFunnelShift(
I);
4052 case Intrinsic::is_constant:
4054 setShadow(&
I, getCleanShadow(&
I));
4055 setOrigin(&
I, getCleanOrigin());
4059 if (!handleUnknownIntrinsic(
I))
4060 visitInstruction(
I);
4065 void visitLibAtomicLoad(
CallBase &CB) {
4067 assert(isa<CallInst>(CB));
4076 Value *NewOrdering =
4080 NextNodeIRBuilder NextIRB(&CB);
4081 Value *SrcShadowPtr, *SrcOriginPtr;
4082 std::tie(SrcShadowPtr, SrcOriginPtr) =
4083 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4085 Value *DstShadowPtr =
4086 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4090 NextIRB.CreateMemCpy(DstShadowPtr,
Align(1), SrcShadowPtr,
Align(1),
Size);
4091 if (MS.TrackOrigins) {
4092 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
4094 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
4095 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
4099 void visitLibAtomicStore(
CallBase &CB) {
4106 Value *NewOrdering =
4110 Value *DstShadowPtr =
4128 visitAsmInstruction(CB);
4130 visitInstruction(CB);
4139 case LibFunc_atomic_load:
4140 if (!isa<CallInst>(CB)) {
4141 llvm::errs() <<
"MSAN -- cannot instrument invoke of libatomic load."
4145 visitLibAtomicLoad(CB);
4147 case LibFunc_atomic_store:
4148 visitLibAtomicStore(CB);
4155 if (
auto *Call = dyn_cast<CallInst>(&CB)) {
4156 assert(!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere");
4164 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4166 Call->removeFnAttrs(
B);
4168 Func->removeFnAttrs(
B);
4174 bool MayCheckCall = MS.EagerChecks;
4178 MayCheckCall &= !
Func->getName().starts_with(
"__sanitizer_unaligned_");
4181 unsigned ArgOffset = 0;
4184 if (!
A->getType()->isSized()) {
4185 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is not sized: " << CB <<
"\n");
4193 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4196 insertShadowCheck(
A, &CB);
4197 Size =
DL.getTypeAllocSize(
A->getType());
4203 Value *ArgShadow = getShadow(
A);
4204 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4206 <<
" Shadow: " << *ArgShadow <<
"\n");
4210 assert(
A->getType()->isPointerTy() &&
4211 "ByVal argument is not a pointer!");
4219 Value *AShadowPtr, *AOriginPtr;
4220 std::tie(AShadowPtr, AOriginPtr) =
4221 getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(), Alignment,
4223 if (!PropagateShadow) {
4230 if (MS.TrackOrigins) {
4231 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4245 Size =
DL.getTypeAllocSize(
A->getType());
4250 Constant *Cst = dyn_cast<Constant>(ArgShadow);
4251 if (MS.TrackOrigins && !(Cst && Cst->
isNullValue())) {
4253 getOriginPtrForArgument(IRB, ArgOffset));
4257 assert(Store !=
nullptr);
4266 if (FT->isVarArg()) {
4267 VAHelper->visitCallBase(CB, IRB);
4274 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
4277 if (MayCheckCall && CB.
hasRetAttr(Attribute::NoUndef)) {
4278 setShadow(&CB, getCleanShadow(&CB));
4279 setOrigin(&CB, getCleanOrigin());
4285 Value *
Base = getShadowPtrForRetval(IRBBefore);
4286 IRBBefore.CreateAlignedStore(getCleanShadow(&CB),
Base,
4289 if (isa<CallInst>(CB)) {
4293 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
4298 setShadow(&CB, getCleanShadow(&CB));
4299 setOrigin(&CB, getCleanOrigin());
4306 "Could not find insertion point for retval shadow load");
4309 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4310 getShadowTy(&CB), getShadowPtrForRetval(IRBAfter),
4312 setShadow(&CB, RetvalShadow);
4313 if (MS.TrackOrigins)
4314 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
4315 getOriginPtrForRetval()));
4319 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
4320 RetVal =
I->getOperand(0);
4322 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
4323 return I->isMustTailCall();
4330 Value *RetVal =
I.getReturnValue();
4336 Value *ShadowPtr = getShadowPtrForRetval(IRB);
4337 bool HasNoUndef =
F.hasRetAttribute(Attribute::NoUndef);
4338 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4341 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (
F.getName() ==
"main");
4343 Value *Shadow = getShadow(RetVal);
4344 bool StoreOrigin =
true;
4346 insertShadowCheck(RetVal, &
I);
4347 Shadow = getCleanShadow(RetVal);
4348 StoreOrigin =
false;
4355 if (MS.TrackOrigins && StoreOrigin)
4356 IRB.
CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
4362 if (!PropagateShadow) {
4363 setShadow(&
I, getCleanShadow(&
I));
4364 setOrigin(&
I, getCleanOrigin());
4368 ShadowPHINodes.push_back(&
I);
4369 setShadow(&
I, IRB.
CreatePHI(getShadowTy(&
I),
I.getNumIncomingValues(),
4371 if (MS.TrackOrigins)
4373 &
I, IRB.
CreatePHI(MS.OriginTy,
I.getNumIncomingValues(),
"_msphi_o"));
4390 IRB.
CreateCall(MS.MsanPoisonStackFn, {&I, Len});
4392 Value *ShadowBase, *OriginBase;
4393 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4400 if (PoisonStack && MS.TrackOrigins) {
4401 Value *Idptr = getLocalVarIdptr(
I);
4403 Value *Descr = getLocalVarDescription(
I);
4404 IRB.
CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4405 {&I, Len, Idptr, Descr});
4407 IRB.
CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
4413 Value *Descr = getLocalVarDescription(
I);
4415 IRB.
CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
4417 IRB.
CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
4424 NextNodeIRBuilder IRB(InsPoint);
4428 if (
I.isArrayAllocation())
4432 if (MS.CompileKernel)
4433 poisonAllocaKmsan(
I, IRB, Len);
4435 poisonAllocaUserspace(
I, IRB, Len);
4439 setShadow(&
I, getCleanShadow(&
I));
4440 setOrigin(&
I, getCleanOrigin());
4452 Value *Sb = getShadow(
B);
4453 Value *Sc = getShadow(
C);
4454 Value *Sd = getShadow(
D);
4459 if (
I.getType()->isAggregateType()) {
4463 Sa1 = getPoisonedShadow(getShadowTy(
I.getType()));
4471 C = CreateAppToShadowCast(IRB,
C);
4472 D = CreateAppToShadowCast(IRB,
D);
4479 if (MS.TrackOrigins) {
4482 if (
B->getType()->isVectorTy()) {
4483 B = convertToBool(
B, IRB);
4484 Sb = convertToBool(Sb, IRB);
4491 getOrigin(
I.getFalseValue()))));
4498 setShadow(&
I, getCleanShadow(&
I));
4499 setOrigin(&
I, getCleanOrigin());
4503 setShadow(&
I, getCleanShadow(&
I));
4504 setOrigin(&
I, getCleanOrigin());
4508 setShadow(&
I, getCleanShadow(&
I));
4509 setOrigin(&
I, getCleanOrigin());
4516 Value *Agg =
I.getAggregateOperand();
4518 Value *AggShadow = getShadow(Agg);
4522 setShadow(&
I, ResShadow);
4523 setOriginForNaryOp(
I);
4529 Value *AggShadow = getShadow(
I.getAggregateOperand());
4530 Value *InsShadow = getShadow(
I.getInsertedValueOperand());
4536 setOriginForNaryOp(
I);
4540 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
4541 errs() <<
"ZZZ call " << CI->getCalledFunction()->getName() <<
"\n";
4543 errs() <<
"ZZZ " <<
I.getOpcodeName() <<
"\n";
4545 errs() <<
"QQQ " <<
I <<
"\n";
4572 insertShadowCheck(Operand, &
I);
4579 auto Size =
DL.getTypeStoreSize(ElemTy);
4581 if (MS.CompileKernel) {
4582 IRB.
CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal});
4588 auto [ShadowPtr,
_] =
4589 getShadowOriginPtrUserspace(Operand, IRB, IRB.
getInt8Ty(),
Align(1));
4600 int NumRetOutputs = 0;
4602 Type *
RetTy = cast<Value>(CB)->getType();
4603 if (!
RetTy->isVoidTy()) {
4605 auto *
ST = dyn_cast<StructType>(
RetTy);
4607 NumRetOutputs =
ST->getNumElements();
4613 switch (
Info.Type) {
4621 return NumOutputs - NumRetOutputs;
4644 int OutputArgs = getNumOutputArgs(IA, CB);
4650 for (
int i = OutputArgs; i < NumOperands; i++) {
4658 for (
int i = 0; i < OutputArgs; i++) {
4664 setShadow(&
I, getCleanShadow(&
I));
4665 setOrigin(&
I, getCleanOrigin());
4670 setShadow(&
I, getCleanShadow(&
I));
4671 setOrigin(&
I, getCleanOrigin());
4679 for (
size_t i = 0, n =
I.getNumOperands(); i < n; i++) {
4680 Value *Operand =
I.getOperand(i);
4682 insertShadowCheck(Operand, &
I);
4684 setShadow(&
I, getCleanShadow(&
I));
4685 setOrigin(&
I, getCleanOrigin());
4689struct VarArgHelperBase :
public VarArgHelper {
4691 MemorySanitizer &MS;
4692 MemorySanitizerVisitor &MSV;
4694 const unsigned VAListTagSize;
4696 VarArgHelperBase(
Function &
F, MemorySanitizer &MS,
4697 MemorySanitizerVisitor &MSV,
unsigned VAListTagSize)
4698 :
F(
F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
4702 return IRB.
CreateAdd(
Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4707 unsigned ArgOffset) {
4716 unsigned ArgOffset,
unsigned ArgSize) {
4720 return getShadowPtrForVAArgument(Ty, IRB, ArgOffset);
4735 unsigned BaseOffset) {
4744 TailSize,
Align(8));
4749 Value *VAListTag =
I.getArgOperand(0);
4751 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
4752 VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
true);
4755 VAListTagSize, Alignment,
false);
4762 unpoisonVAListTagForInst(
I);
4768 unpoisonVAListTagForInst(
I);
4773struct VarArgAMD64Helper :
public VarArgHelperBase {
4776 static const unsigned AMD64GpEndOffset = 48;
4777 static const unsigned AMD64FpEndOffsetSSE = 176;
4779 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4781 unsigned AMD64FpEndOffset;
4784 Value *VAArgOverflowSize =
nullptr;
4786 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4788 VarArgAMD64Helper(
Function &
F, MemorySanitizer &MS,
4789 MemorySanitizerVisitor &MSV)
4790 : VarArgHelperBase(
F, MS, MSV, 24) {
4791 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4792 for (
const auto &Attr :
F.getAttributes().getFnAttrs()) {
4793 if (Attr.isStringAttribute() &&
4794 (Attr.getKindAsString() ==
"target-features")) {
4795 if (Attr.getValueAsString().contains(
"-sse"))
4796 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4802 ArgKind classifyArgument(
Value *arg) {
4805 if (
T->isX86_FP80Ty())
4807 if (
T->isFPOrFPVectorTy() ||
T->isX86_MMXTy())
4808 return AK_FloatingPoint;
4809 if (
T->isIntegerTy() &&
T->getPrimitiveSizeInBits() <= 64)
4810 return AK_GeneralPurpose;
4811 if (
T->isPointerTy())
4812 return AK_GeneralPurpose;
4825 unsigned GpOffset = 0;
4826 unsigned FpOffset = AMD64GpEndOffset;
4827 unsigned OverflowOffset = AMD64FpEndOffset;
4832 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
4839 assert(
A->getType()->isPointerTy());
4841 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
4843 unsigned BaseOffset = OverflowOffset;
4845 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
4846 Value *OriginBase =
nullptr;
4847 if (MS.TrackOrigins)
4848 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4849 OverflowOffset += AlignedSize;
4852 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4856 Value *ShadowPtr, *OriginPtr;
4857 std::tie(ShadowPtr, OriginPtr) =
4862 if (MS.TrackOrigins)
4866 ArgKind AK = classifyArgument(
A);
4867 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4869 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4871 Value *ShadowBase, *OriginBase =
nullptr;
4873 case AK_GeneralPurpose:
4874 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, GpOffset);
4875 if (MS.TrackOrigins)
4876 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset);
4880 case AK_FloatingPoint:
4881 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, FpOffset);
4882 if (MS.TrackOrigins)
4883 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
4890 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
4892 unsigned BaseOffset = OverflowOffset;
4894 getShadowPtrForVAArgument(
A->getType(), IRB, OverflowOffset);
4895 if (MS.TrackOrigins) {
4896 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4898 OverflowOffset += AlignedSize;
4901 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4910 Value *Shadow = MSV.getShadow(
A);
4912 if (MS.TrackOrigins) {
4913 Value *Origin = MSV.getOrigin(
A);
4915 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4921 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
4922 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4925 void finalizeInstrumentation()
override {
4926 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4927 "finalizeInstrumentation called twice");
4928 if (!VAStartInstrumentationList.
empty()) {
4935 ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize);
4942 Intrinsic::umin, CopySize,
4946 if (MS.TrackOrigins) {
4956 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
4957 CallInst *OrigInst = VAStartInstrumentationList[i];
4958 NextNodeIRBuilder IRB(OrigInst);
4961 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
4964 ConstantInt::get(MS.IntptrTy, 16)),
4965 PointerType::get(RegSaveAreaPtrTy, 0));
4966 Value *RegSaveAreaPtr =
4967 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4968 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4970 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4971 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
4973 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4975 if (MS.TrackOrigins)
4976 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
4977 Alignment, AMD64FpEndOffset);
4978 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
4981 ConstantInt::get(MS.IntptrTy, 8)),
4982 PointerType::get(OverflowArgAreaPtrTy, 0));
4983 Value *OverflowArgAreaPtr =
4984 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
4985 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
4986 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
4987 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
4991 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
4993 if (MS.TrackOrigins) {
4996 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5005struct VarArgMIPS64Helper :
public VarArgHelperBase {
5007 Value *VAArgSize =
nullptr;
5009 VarArgMIPS64Helper(
Function &
F, MemorySanitizer &MS,
5010 MemorySanitizerVisitor &MSV)
5011 : VarArgHelperBase(
F, MS, MSV, 8) {}
5014 unsigned VAArgOffset = 0;
5018 Triple TargetTriple(
F.getParent()->getTargetTriple());
5020 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5025 VAArgOffset += (8 - ArgSize);
5027 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VAArgOffset, ArgSize);
5028 VAArgOffset += ArgSize;
5029 VAArgOffset =
alignTo(VAArgOffset, 8);
5038 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5041 void finalizeInstrumentation()
override {
5042 assert(!VAArgSize && !VAArgTLSCopy &&
5043 "finalizeInstrumentation called twice");
5047 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5049 if (!VAStartInstrumentationList.
empty()) {
5058 Intrinsic::umin, CopySize,
5066 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5067 CallInst *OrigInst = VAStartInstrumentationList[i];
5068 NextNodeIRBuilder IRB(OrigInst);
5070 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5071 Value *RegSaveAreaPtrPtr =
5073 PointerType::get(RegSaveAreaPtrTy, 0));
5074 Value *RegSaveAreaPtr =
5075 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5076 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5078 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5079 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5081 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5088struct VarArgAArch64Helper :
public VarArgHelperBase {
5089 static const unsigned kAArch64GrArgSize = 64;
5090 static const unsigned kAArch64VrArgSize = 128;
5092 static const unsigned AArch64GrBegOffset = 0;
5093 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
5095 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
5096 static const unsigned AArch64VrEndOffset =
5097 AArch64VrBegOffset + kAArch64VrArgSize;
5098 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
5101 Value *VAArgOverflowSize =
nullptr;
5103 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
5105 VarArgAArch64Helper(
Function &
F, MemorySanitizer &MS,
5106 MemorySanitizerVisitor &MSV)
5107 : VarArgHelperBase(
F, MS, MSV, 32) {}
5110 std::pair<ArgKind, uint64_t> classifyArgument(
Type *
T) {
5111 if (
T->isIntOrPtrTy() &&
T->getPrimitiveSizeInBits() <= 64)
5112 return {AK_GeneralPurpose, 1};
5113 if (
T->isFloatingPointTy() &&
T->getPrimitiveSizeInBits() <= 128)
5114 return {AK_FloatingPoint, 1};
5116 if (
T->isArrayTy()) {
5117 auto R = classifyArgument(
T->getArrayElementType());
5118 R.second *=
T->getScalarType()->getArrayNumElements();
5123 auto R = classifyArgument(FV->getScalarType());
5124 R.second *= FV->getNumElements();
5129 return {AK_Memory, 0};
5142 unsigned GrOffset = AArch64GrBegOffset;
5143 unsigned VrOffset = AArch64VrBegOffset;
5144 unsigned OverflowOffset = AArch64VAEndOffset;
5149 auto [AK, RegNum] = classifyArgument(
A->getType());
5150 if (AK == AK_GeneralPurpose &&
5151 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
5153 if (AK == AK_FloatingPoint &&
5154 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
5158 case AK_GeneralPurpose:
5159 Base = getShadowPtrForVAArgument(
A->getType(), IRB, GrOffset);
5160 GrOffset += 8 * RegNum;
5162 case AK_FloatingPoint:
5163 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VrOffset);
5164 VrOffset += 16 * RegNum;
5171 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5173 unsigned BaseOffset = OverflowOffset;
5174 Base = getShadowPtrForVAArgument(
A->getType(), IRB, BaseOffset);
5175 OverflowOffset += AlignedSize;
5178 CleanUnusedTLS(IRB,
Base, BaseOffset);
5190 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
5191 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5198 ConstantInt::get(MS.IntptrTy, offset)),
5199 PointerType::get(*MS.C, 0));
5207 ConstantInt::get(MS.IntptrTy, offset)),
5208 PointerType::get(*MS.C, 0));
5210 return IRB.
CreateSExt(SaveArea32, MS.IntptrTy);
5213 void finalizeInstrumentation()
override {
5214 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5215 "finalizeInstrumentation called twice");
5216 if (!VAStartInstrumentationList.
empty()) {
5223 ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize);
5230 Intrinsic::umin, CopySize,
5236 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
5237 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
5241 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5242 CallInst *OrigInst = VAStartInstrumentationList[i];
5243 NextNodeIRBuilder IRB(OrigInst);
5262 Value *StackSaveAreaPtr =
5263 IRB.
CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy);
5266 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
5267 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
5270 IRB.
CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy);
5273 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
5274 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
5277 IRB.
CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy);
5283 Value *GrRegSaveAreaShadowPtrOff =
5284 IRB.
CreateAdd(GrArgSize, GrOffSaveArea);
5286 Value *GrRegSaveAreaShadowPtr =
5287 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5293 Value *GrCopySize = IRB.
CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
5299 Value *VrRegSaveAreaShadowPtrOff =
5300 IRB.
CreateAdd(VrArgSize, VrOffSaveArea);
5302 Value *VrRegSaveAreaShadowPtr =
5303 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5310 VrRegSaveAreaShadowPtrOff);
5311 Value *VrCopySize = IRB.
CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
5317 Value *StackSaveAreaShadowPtr =
5318 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5323 VAArgTLSCopy, IRB.
getInt32(AArch64VAEndOffset));
5326 Align(16), VAArgOverflowSize);
5332struct VarArgPowerPC64Helper :
public VarArgHelperBase {
5334 Value *VAArgSize =
nullptr;
5336 VarArgPowerPC64Helper(
Function &
F, MemorySanitizer &MS,
5337 MemorySanitizerVisitor &MSV)
5338 : VarArgHelperBase(
F, MS, MSV, 8) {}
5348 Triple TargetTriple(
F.getParent()->getTargetTriple());
5356 unsigned VAArgOffset = VAArgBase;
5360 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
5362 assert(
A->getType()->isPointerTy());
5364 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
5367 ArgAlign =
Align(8);
5368 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5370 Value *
Base = getShadowPtrForVAArgument(
5371 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
5373 Value *AShadowPtr, *AOriginPtr;
5374 std::tie(AShadowPtr, AOriginPtr) =
5375 MSV.getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(),
5385 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5387 if (
A->getType()->isArrayTy()) {
5390 Type *ElementTy =
A->getType()->getArrayElementType();
5392 ArgAlign =
Align(
DL.getTypeAllocSize(ElementTy));
5393 }
else if (
A->getType()->isVectorTy()) {
5395 ArgAlign =
Align(ArgSize);
5398 ArgAlign =
Align(8);
5399 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5400 if (
DL.isBigEndian()) {
5404 VAArgOffset += (8 - ArgSize);
5407 Base = getShadowPtrForVAArgument(
A->getType(), IRB,
5408 VAArgOffset - VAArgBase, ArgSize);
5412 VAArgOffset += ArgSize;
5416 VAArgBase = VAArgOffset;
5420 ConstantInt::get(IRB.
getInt64Ty(), VAArgOffset - VAArgBase);
5423 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5426 void finalizeInstrumentation()
override {
5427 assert(!VAArgSize && !VAArgTLSCopy &&
5428 "finalizeInstrumentation called twice");
5432 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5434 if (!VAStartInstrumentationList.
empty()) {
5444 Intrinsic::umin, CopySize,
5452 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5453 CallInst *OrigInst = VAStartInstrumentationList[i];
5454 NextNodeIRBuilder IRB(OrigInst);
5456 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5457 Value *RegSaveAreaPtrPtr =
5459 PointerType::get(RegSaveAreaPtrTy, 0));
5460 Value *RegSaveAreaPtr =
5461 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5462 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5464 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5465 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5467 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5474struct VarArgSystemZHelper :
public VarArgHelperBase {
5475 static const unsigned SystemZGpOffset = 16;
5476 static const unsigned SystemZGpEndOffset = 56;
5477 static const unsigned SystemZFpOffset = 128;
5478 static const unsigned SystemZFpEndOffset = 160;
5479 static const unsigned SystemZMaxVrArgs = 8;
5480 static const unsigned SystemZRegSaveAreaSize = 160;
5481 static const unsigned SystemZOverflowOffset = 160;
5482 static const unsigned SystemZVAListTagSize = 32;
5483 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5484 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5486 bool IsSoftFloatABI;
5489 Value *VAArgOverflowSize =
nullptr;
5491 enum class ArgKind {
5499 enum class ShadowExtension {
None,
Zero, Sign };
5501 VarArgSystemZHelper(
Function &
F, MemorySanitizer &MS,
5502 MemorySanitizerVisitor &MSV)
5503 : VarArgHelperBase(
F, MS, MSV, SystemZVAListTagSize),
5504 IsSoftFloatABI(
F.getFnAttribute(
"use-soft-float").getValueAsBool()) {}
5506 ArgKind classifyArgument(
Type *
T) {
5513 if (
T->isIntegerTy(128) ||
T->isFP128Ty())
5514 return ArgKind::Indirect;
5515 if (
T->isFloatingPointTy())
5516 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5517 if (
T->isIntegerTy() ||
T->isPointerTy())
5518 return ArgKind::GeneralPurpose;
5519 if (
T->isVectorTy())
5520 return ArgKind::Vector;
5521 return ArgKind::Memory;
5524 ShadowExtension getShadowExtension(
const CallBase &CB,
unsigned ArgNo) {
5534 return ShadowExtension::Zero;
5538 return ShadowExtension::Sign;
5540 return ShadowExtension::None;
5544 unsigned GpOffset = SystemZGpOffset;
5545 unsigned FpOffset = SystemZFpOffset;
5546 unsigned VrIndex = 0;
5547 unsigned OverflowOffset = SystemZOverflowOffset;
5554 ArgKind AK = classifyArgument(
T);
5555 if (AK == ArgKind::Indirect) {
5556 T = PointerType::get(
T, 0);
5557 AK = ArgKind::GeneralPurpose;
5559 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5560 AK = ArgKind::Memory;
5561 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5562 AK = ArgKind::Memory;
5563 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5564 AK = ArgKind::Memory;
5565 Value *ShadowBase =
nullptr;
5566 Value *OriginBase =
nullptr;
5567 ShadowExtension SE = ShadowExtension::None;
5569 case ArgKind::GeneralPurpose: {
5574 SE = getShadowExtension(CB, ArgNo);
5576 if (SE == ShadowExtension::None) {
5578 assert(ArgAllocSize <= ArgSize);
5579 GapSize = ArgSize - ArgAllocSize;
5581 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5582 if (MS.TrackOrigins)
5583 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5585 GpOffset += ArgSize;
5591 case ArgKind::FloatingPoint: {
5600 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5601 if (MS.TrackOrigins)
5602 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5604 FpOffset += ArgSize;
5610 case ArgKind::Vector: {
5617 case ArgKind::Memory: {
5625 SE = getShadowExtension(CB, ArgNo);
5627 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5629 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5630 if (MS.TrackOrigins)
5632 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5633 OverflowOffset += ArgSize;
5640 case ArgKind::Indirect:
5643 if (ShadowBase ==
nullptr)
5645 Value *Shadow = MSV.getShadow(
A);
5646 if (SE != ShadowExtension::None)
5647 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.
getInt64Ty(),
5648 SE == ShadowExtension::Sign);
5650 ShadowBase, PointerType::get(Shadow->
getType(), 0),
"_msarg_va_s");
5652 if (MS.TrackOrigins) {
5653 Value *Origin = MSV.getOrigin(
A);
5655 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5659 Constant *OverflowSize = ConstantInt::get(
5660 IRB.
getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5661 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5665 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5669 ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5670 PointerType::get(RegSaveAreaPtrTy, 0));
5671 Value *RegSaveAreaPtr = IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5672 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5674 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5675 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(), Alignment,
5680 unsigned RegSaveAreaSize =
5681 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
5682 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5684 if (MS.TrackOrigins)
5685 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5686 Alignment, RegSaveAreaSize);
5692 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5696 ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5697 PointerType::get(OverflowArgAreaPtrTy, 0));
5698 Value *OverflowArgAreaPtr =
5699 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5700 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5702 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5703 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5706 SystemZOverflowOffset);
5707 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5709 if (MS.TrackOrigins) {
5711 SystemZOverflowOffset);
5712 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5717 void finalizeInstrumentation()
override {
5718 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5719 "finalizeInstrumentation called twice");
5720 if (!VAStartInstrumentationList.
empty()) {
5727 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5735 Intrinsic::umin, CopySize,
5739 if (MS.TrackOrigins) {
5749 for (
size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.
size();
5750 VaStartNo < VaStartNum; VaStartNo++) {
5751 CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5752 NextNodeIRBuilder IRB(OrigInst);
5754 copyRegSaveArea(IRB, VAListTag);
5755 copyOverflowArea(IRB, VAListTag);
5762using VarArgLoongArch64Helper = VarArgMIPS64Helper;
5765struct VarArgNoOpHelper :
public VarArgHelper {
5766 VarArgNoOpHelper(
Function &
F, MemorySanitizer &MS,
5767 MemorySanitizerVisitor &MSV) {}
5775 void finalizeInstrumentation()
override {}
5781 MemorySanitizerVisitor &Visitor) {
5784 Triple TargetTriple(Func.getParent()->getTargetTriple());
5786 return new VarArgAMD64Helper(Func, Msan, Visitor);
5788 return new VarArgMIPS64Helper(Func, Msan, Visitor);
5790 return new VarArgAArch64Helper(Func, Msan, Visitor);
5793 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5795 return new VarArgSystemZHelper(Func, Msan, Visitor);
5797 return new VarArgLoongArch64Helper(Func, Msan, Visitor);
5799 return new VarArgNoOpHelper(Func, Msan, Visitor);
5806 if (
F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
5809 MemorySanitizerVisitor Visitor(
F, *
this, TLI);
5813 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
5816 return Visitor.runOnFunction();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
static bool isStore(int Opcode)
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
static const size_t kNumberOfAccessSizes
VarLocInsertPt getNextNode(const DbgRecord *DVR)
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AtomicOrdering addReleaseOrdering(AtomicOrdering AO)
static AtomicOrdering addAcquireOrdering(AtomicOrdering AO)
static bool isAMustTailRetVal(Value *RetVal)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool runOnFunction(Function &F, bool PostInlining)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
static size_t TypeSizeToSizeIndex(uint32_t TypeSize)
static const MemoryMapParams Linux_LoongArch64_MemoryMapParams
static const PlatformMemoryMapParams Linux_S390_MemoryMapParams
static const Align kMinOriginAlignment
static const MemoryMapParams Linux_X86_64_MemoryMapParams
static cl::opt< uint64_t > ClShadowBase("msan-shadow-base", cl::desc("Define custom MSan ShadowBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClDumpStrictInstructions("msan-dump-strict-instructions", cl::desc("print out instructions with default strict semantics"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClHandleICmpExact("msan-handle-icmp-exact", cl::desc("exact handling of relational integer ICmp"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams Linux_X86_MemoryMapParams
static cl::opt< uint64_t > ClOriginBase("msan-origin-base", cl::desc("Define custom MSan OriginBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClCheckConstantShadow("msan-check-constant-shadow", cl::desc("Insert checks for constant shadow values"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams
static const MemoryMapParams NetBSD_X86_64_MemoryMapParams
static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams
static const unsigned kOriginSize
static cl::opt< bool > ClWithComdat("msan-with-comdat", cl::desc("Place MSan constructors in comdat sections"), cl::Hidden, cl::init(false))
static cl::opt< int > ClTrackOrigins("msan-track-origins", cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden, cl::init(0))
Track origins of uninitialized values.
static cl::opt< int > ClInstrumentationWithCallThreshold("msan-instrumentation-with-call-threshold", cl::desc("If the function being instrumented requires more than " "this number of checks and origin stores, use callbacks instead of " "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500))
static cl::opt< int > ClPoisonStackPattern("msan-poison-stack-pattern", cl::desc("poison uninitialized stack variables with the given pattern"), cl::Hidden, cl::init(0xff))
static const Align kShadowTLSAlignment
static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams
static Constant * getOrInsertGlobal(Module &M, StringRef Name, Type *Ty)
static const MemoryMapParams Linux_S390X_MemoryMapParams
static cl::opt< bool > ClPoisonUndef("msan-poison-undef", cl::desc("poison undef temps"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClPoisonStack("msan-poison-stack", cl::desc("poison uninitialized stack variables"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_I386_MemoryMapParams
const char kMsanInitName[]
static cl::opt< bool > ClPrintStackNames("msan-print-stack-names", cl::desc("Print name of local stack variable"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_AArch64_MemoryMapParams
static cl::opt< uint64_t > ClAndMask("msan-and-mask", cl::desc("Define custom MSan AndMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleLifetimeIntrinsics("msan-handle-lifetime-intrinsics", cl::desc("when possible, poison scoped variables at the beginning of the scope " "(slower, but more precise)"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClKeepGoing("msan-keep-going", cl::desc("keep going after reporting a UMR"), cl::Hidden, cl::init(false))
static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams
static GlobalVariable * createPrivateConstGlobalForString(Module &M, StringRef Str)
Create a non-const global initialized with the given string.
static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams
static const size_t kNumberOfAccessSizes
static cl::opt< bool > ClEagerChecks("msan-eager-checks", cl::desc("check arguments and return values at function call boundaries"), cl::Hidden, cl::init(false))
static cl::opt< int > ClDisambiguateWarning("msan-disambiguate-warning-threshold", cl::desc("Define threshold for number of checks per " "debug location to force origin update."), cl::Hidden, cl::init(3))
static VarArgHelper * CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, MemorySanitizerVisitor &Visitor)
static const MemoryMapParams Linux_MIPS64_MemoryMapParams
static const MemoryMapParams Linux_PowerPC64_MemoryMapParams
static cl::opt< uint64_t > ClXorMask("msan-xor-mask", cl::desc("Define custom MSan XorMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleAsmConservative("msan-handle-asm-conservative", cl::desc("conservative handling of inline assembly"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams
static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams
static const unsigned kParamTLSSize
static cl::opt< bool > ClHandleICmp("msan-handle-icmp", cl::desc("propagate shadow through ICmpEQ and ICmpNE"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClEnableKmsan("msan-kernel", cl::desc("Enable KernelMemorySanitizer instrumentation"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClPoisonStackWithCall("msan-poison-stack-with-call", cl::desc("poison uninitialized stack variables with a call"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams
static const unsigned kRetvalTLSSize
static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams
const char kMsanModuleCtorName[]
static const MemoryMapParams FreeBSD_I386_MemoryMapParams
static cl::opt< bool > ClCheckAccessAddress("msan-check-access-address", cl::desc("report accesses through a pointer which has poisoned shadow"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClDisableChecks("msan-disable-checks", cl::desc("Apply no_sanitize to the whole file"), cl::Hidden, cl::init(false))
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
InstListType::iterator iterator
Instruction iterators...
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Type * getParamByValType(unsigned ArgNo) const
Extract the byval type for a call or parameter.
Value * getCalledOperand() const
Type * getParamElementType(unsigned ArgNo) const
Extract the elementtype type for a parameter.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_SGT
signed greater than
@ ICMP_SGE
signed greater or equal
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
bool isZeroValue() const
Return true if the value is negative zero or null value.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
static bool shouldExecute(unsigned CounterName)
This instruction compares its operands according to the predicate given to the constructor.
Class to represent fixed width SIMD vectors.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void setComdat(Comdat *C)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ ExternalLinkage
Externally visible function.
Analysis pass providing a never-invalidated alias analysis result.
This instruction compares its operands according to the predicate given to the constructor.
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTypeSize(Type *DstType, TypeSize Size)
Create an expression which evaluates to the number of units in Size at runtime.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::vector< ConstraintInfo > ConstraintInfoVector
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
This class represents a cast from an integer to a pointer.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
This class wraps the llvm.memcpy intrinsic.
This class wraps the llvm.memmove intrinsic.
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
This class represents a cast from a pointer to an integer.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This instruction constructs a fixed permutation of two input vectors.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
AttributeList getAttrList(LLVMContext *C, ArrayRef< unsigned > ArgNos, bool Signed, bool Ret=false, AttributeList AL=AttributeList()) const
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Triple - Helper class for working with autoconf configuration names.
bool isMIPS64() const
Tests whether the target is MIPS 64-bit (little and big endian).
ArchType getArch() const
Get the parsed architecture type of this triple.
bool isLoongArch64() const
Tests whether the target is 64-bit LoongArch.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
static Type * getX86_MMXTy(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
'undef' values are things that do not have specified contents.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This represents the llvm.va_copy intrinsic.
This represents the llvm.va_start intrinsic.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
This class represents zero extension of integer types.
int getNumOccurrences() const
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
This class provides various memory handling functions that manipulate MemoryBlock instances.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
initializer< Ty > init(const Ty &Val)
Function * Kernel
Summary of a kernel (=entry point for target offloading).
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
void stable_sort(R &&Range)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
AllocaInst * findAllocaForValue(Value *V, bool OffsetZero=false)
Returns unique alloca where the value comes from, or nullptr.
std::pair< Function *, FunctionCallee > getOrCreateSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, function_ref< void(Function *, FunctionCallee)> FunctionsCreatedCallback, StringRef VersionCheckName=StringRef(), bool Weak=false)
Creates sanitizer constructor function lazily.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Or
Bitwise or logical OR of integers.
std::pair< Instruction *, Value * > SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore)
Insert a for (int i = 0; i < End; i++) loop structure (with the exception that End is assumed > 0,...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
iterator_range< df_iterator< T > > depth_first(const T &G)
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, const TargetLibraryInfo *TLI)
Given a CallInst, check if it calls a string function known to CodeGen, and mark it with NoBuiltin if...
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A CRTP mix-in to automatically provide informational APIs needed for passes.