181#include "llvm/IR/IntrinsicsX86.h"
210#define DEBUG_TYPE "msan"
213 "Controls which checks to insert");
231 "msan-track-origins",
236 cl::desc(
"keep going after reporting a UMR"),
245 "msan-poison-stack-with-call",
250 "msan-poison-stack-pattern",
251 cl::desc(
"poison uninitialized stack variables with the given pattern"),
256 cl::desc(
"Print name of local stack variable"),
265 cl::desc(
"propagate shadow through ICmpEQ and ICmpNE"),
270 cl::desc(
"exact handling of relational integer ICmp"),
274 "msan-handle-lifetime-intrinsics",
276 "when possible, poison scoped variables at the beginning of the scope "
277 "(slower, but more precise)"),
288 "msan-handle-asm-conservative",
299 "msan-check-access-address",
300 cl::desc(
"report accesses through a pointer which has poisoned shadow"),
305 cl::desc(
"check arguments and return values at function call boundaries"),
309 "msan-dump-strict-instructions",
310 cl::desc(
"print out instructions with default strict semantics"),
314 "msan-instrumentation-with-call-threshold",
316 "If the function being instrumented requires more than "
317 "this number of checks and origin stores, use callbacks instead of "
318 "inline checks (-1 means never use callbacks)."),
323 cl::desc(
"Enable KernelMemorySanitizer instrumentation"),
333 cl::desc(
"Insert checks for constant shadow values"),
340 cl::desc(
"Place MSan constructors in comdat sections"),
346 cl::desc(
"Define custom MSan AndMask"),
350 cl::desc(
"Define custom MSan XorMask"),
354 cl::desc(
"Define custom MSan ShadowBase"),
358 cl::desc(
"Define custom MSan OriginBase"),
363 cl::desc(
"Define threshold for number of checks per "
364 "debug location to force origin update."),
376struct MemoryMapParams {
383struct PlatformMemoryMapParams {
384 const MemoryMapParams *bits32;
385 const MemoryMapParams *bits64;
531class MemorySanitizer {
540 MemorySanitizer(MemorySanitizer &&) =
delete;
541 MemorySanitizer &operator=(MemorySanitizer &&) =
delete;
542 MemorySanitizer(
const MemorySanitizer &) =
delete;
543 MemorySanitizer &operator=(
const MemorySanitizer &) =
delete;
548 friend struct MemorySanitizerVisitor;
549 friend struct VarArgHelperBase;
550 friend struct VarArgAMD64Helper;
551 friend struct VarArgMIPS64Helper;
552 friend struct VarArgAArch64Helper;
553 friend struct VarArgPowerPC64Helper;
554 friend struct VarArgSystemZHelper;
556 void initializeModule(
Module &M);
561 template <
typename... ArgsTy>
588 Value *ParamOriginTLS;
594 Value *RetvalOriginTLS;
600 Value *VAArgOriginTLS;
603 Value *VAArgOverflowSizeTLS;
606 bool CallbacksInitialized =
false;
651 Value *MsanMetadataAlloca;
657 const MemoryMapParams *MapParams;
661 MemoryMapParams CustomMapParams;
666 MDNode *OriginStoreWeights;
669void insertModuleCtor(
Module &M) {
697 Recover(getOptOrDefault(
ClKeepGoing, Kernel || R)),
712 MemorySanitizer Msan(*
F.getParent(),
Options);
731 OS, MapClassName2PassName);
738 OS <<
"eager-checks;";
739 OS <<
"track-origins=" <<
Options.TrackOrigins;
755template <
typename... ArgsTy>
763 std::forward<ArgsTy>(Args)...);
766 return M.getOrInsertFunction(
Name, MsanMetadata,
767 std::forward<ArgsTy>(Args)...);
776 RetvalOriginTLS =
nullptr;
778 ParamOriginTLS =
nullptr;
780 VAArgOriginTLS =
nullptr;
781 VAArgOverflowSizeTLS =
nullptr;
783 WarningFn =
M.getOrInsertFunction(
"__msan_warning",
785 IRB.getVoidTy(), IRB.getInt32Ty());
796 MsanGetContextStateFn =
M.getOrInsertFunction(
802 for (
int ind = 0, size = 1; ind < 4; ind++,
size <<= 1) {
803 std::string name_load =
804 "__msan_metadata_ptr_for_load_" + std::to_string(size);
805 std::string name_store =
806 "__msan_metadata_ptr_for_store_" + std::to_string(size);
807 MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction(
809 MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction(
813 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
816 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
817 M,
"__msan_metadata_ptr_for_store_n",
821 MsanPoisonAllocaFn =
M.getOrInsertFunction(
822 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
823 MsanUnpoisonAllocaFn =
M.getOrInsertFunction(
824 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
828 return M.getOrInsertGlobal(
Name, Ty, [&] {
830 nullptr,
Name,
nullptr,
843 StringRef WarningFnName = Recover ?
"__msan_warning_with_origin"
844 :
"__msan_warning_with_origin_noreturn";
845 WarningFn =
M.getOrInsertFunction(WarningFnName,
847 IRB.getVoidTy(), IRB.getInt32Ty());
850 Recover ?
"__msan_warning" :
"__msan_warning_noreturn";
851 WarningFn =
M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
877 VAArgOverflowSizeTLS =
882 unsigned AccessSize = 1 << AccessSizeIndex;
883 std::string FunctionName =
"__msan_maybe_warning_" + itostr(AccessSize);
884 MaybeWarningFn[AccessSizeIndex] =
M.getOrInsertFunction(
886 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
888 FunctionName =
"__msan_maybe_store_origin_" + itostr(AccessSize);
889 MaybeStoreOriginFn[AccessSizeIndex] =
M.getOrInsertFunction(
891 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
895 MsanSetAllocaOriginWithDescriptionFn =
896 M.getOrInsertFunction(
"__msan_set_alloca_origin_with_descr",
897 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
898 MsanSetAllocaOriginNoDescriptionFn =
899 M.getOrInsertFunction(
"__msan_set_alloca_origin_no_descr",
900 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
901 MsanPoisonStackFn =
M.getOrInsertFunction(
"__msan_poison_stack",
902 IRB.getVoidTy(), PtrTy, IntptrTy);
908 if (CallbacksInitialized)
914 MsanChainOriginFn =
M.getOrInsertFunction(
915 "__msan_chain_origin",
918 MsanSetOriginFn =
M.getOrInsertFunction(
920 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
922 M.getOrInsertFunction(
"__msan_memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
924 M.getOrInsertFunction(
"__msan_memcpy", PtrTy, PtrTy, PtrTy, IntptrTy);
925 MemsetFn =
M.getOrInsertFunction(
"__msan_memset",
927 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
929 MsanInstrumentAsmStoreFn =
930 M.getOrInsertFunction(
"__msan_instrument_asm_store", IRB.getVoidTy(),
934 createKernelApi(M, TLI);
936 createUserspaceApi(M, TLI);
938 CallbacksInitialized =
true;
944 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
962void MemorySanitizer::initializeModule(
Module &M) {
963 auto &
DL =
M.getDataLayout();
965 TargetTriple =
Triple(
M.getTargetTriple());
967 bool ShadowPassed =
ClShadowBase.getNumOccurrences() > 0;
968 bool OriginPassed =
ClOriginBase.getNumOccurrences() > 0;
970 if (ShadowPassed || OriginPassed) {
975 MapParams = &CustomMapParams;
977 switch (TargetTriple.getOS()) {
979 switch (TargetTriple.getArch()) {
994 switch (TargetTriple.getArch()) {
1003 switch (TargetTriple.getArch()) {
1037 C = &(
M.getContext());
1039 IntptrTy = IRB.getIntPtrTy(
DL);
1040 OriginTy = IRB.getInt32Ty();
1041 PtrTy = IRB.getPtrTy();
1046 if (!CompileKernel) {
1048 M.getOrInsertGlobal(
"__msan_track_origins", IRB.getInt32Ty(), [&] {
1049 return new GlobalVariable(
1050 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1051 IRB.getInt32(TrackOrigins),
"__msan_track_origins");
1055 M.getOrInsertGlobal(
"__msan_keep_going", IRB.getInt32Ty(), [&] {
1056 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1057 GlobalValue::WeakODRLinkage,
1058 IRB.getInt32(Recover),
"__msan_keep_going");
1073struct VarArgHelper {
1074 virtual ~VarArgHelper() =
default;
1089 virtual void finalizeInstrumentation() = 0;
1092struct MemorySanitizerVisitor;
1097 MemorySanitizerVisitor &Visitor);
1104 if (TypeSizeFixed <= 8)
1113class NextNodeIRBuilder :
public IRBuilder<> {
1126struct MemorySanitizerVisitor :
public InstVisitor<MemorySanitizerVisitor> {
1128 MemorySanitizer &MS;
1131 std::unique_ptr<VarArgHelper> VAHelper;
1138 bool PropagateShadow;
1142 struct ShadowOriginAndInsertPoint {
1148 : Shadow(S), Origin(
O), OrigIns(
I) {}
1156 int64_t SplittableBlocksCount = 0;
1158 MemorySanitizerVisitor(
Function &
F, MemorySanitizer &MS,
1161 bool SanitizeFunction =
1163 InsertChecks = SanitizeFunction;
1164 PropagateShadow = SanitizeFunction;
1174 MS.initializeCallbacks(*
F.getParent(), TLI);
1175 FnPrologueEnd =
IRBuilder<>(
F.getEntryBlock().getFirstNonPHI())
1178 if (MS.CompileKernel) {
1180 insertKmsanPrologue(IRB);
1184 <<
"MemorySanitizer is not inserting checks into '"
1185 <<
F.getName() <<
"'\n");
1188 bool instrumentWithCalls(
Value *V) {
1190 if (isa<Constant>(V))
1193 ++SplittableBlocksCount;
1199 return I.getParent() == FnPrologueEnd->
getParent() &&
1200 (&
I == FnPrologueEnd ||
I.comesBefore(FnPrologueEnd));
1208 if (MS.TrackOrigins <= 1)
1210 return IRB.
CreateCall(MS.MsanChainOriginFn, V);
1215 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1227 const Align IntptrAlignment =
DL.getABITypeAlign(MS.IntptrTy);
1228 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1238 auto [InsertPt,
Index] =
1250 Align CurrentAlignment = Alignment;
1251 if (Alignment >= IntptrAlignment && IntptrSize >
kOriginSize) {
1252 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1253 Value *IntptrOriginPtr =
1255 for (
unsigned i = 0; i <
Size / IntptrSize; ++i) {
1260 CurrentAlignment = IntptrAlignment;
1277 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1278 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1286 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1295 if (instrumentWithCalls(ConvertedShadow) &&
1298 Value *ConvertedShadow2 =
1304 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1308 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1313 void materializeStores() {
1316 Value *Val =
SI->getValueOperand();
1318 Value *Shadow =
SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1319 Value *ShadowPtr, *OriginPtr;
1321 const Align Alignment =
SI->getAlign();
1323 std::tie(ShadowPtr, OriginPtr) =
1324 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
true);
1333 if (MS.TrackOrigins && !
SI->isAtomic())
1334 storeOrigin(IRB,
Addr, Shadow, getOrigin(Val), OriginPtr,
1341 if (MS.TrackOrigins < 2)
1344 if (LazyWarningDebugLocationCount.
empty())
1345 for (
const auto &
I : InstrumentationList)
1346 ++LazyWarningDebugLocationCount[
I.OrigIns->getDebugLoc()];
1360 if (
Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1362 auto NewDebugLoc = OI->getDebugLoc();
1369 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1370 Origin = updateOrigin(Origin, IRBOrigin);
1375 if (MS.CompileKernel || MS.TrackOrigins)
1389 if (instrumentWithCalls(ConvertedShadow) &&
1392 Value *ConvertedShadow2 =
1395 Fn, {ConvertedShadow2,
1396 MS.TrackOrigins && Origin ? Origin : (
Value *)IRB.
getInt32(0)});
1400 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1403 !MS.Recover, MS.ColdCallWeights);
1406 insertWarningFn(IRB, Origin);
1411 void materializeInstructionChecks(
1416 bool Combine = !MS.TrackOrigins;
1418 Value *Shadow =
nullptr;
1419 for (
const auto &ShadowData : InstructionChecks) {
1423 Value *ConvertedShadow = ShadowData.Shadow;
1425 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1432 insertWarningFn(IRB, ShadowData.Origin);
1442 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1447 Shadow = ConvertedShadow;
1451 Shadow = convertToBool(Shadow, IRB,
"_mscmp");
1452 ConvertedShadow = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1453 Shadow = IRB.
CreateOr(Shadow, ConvertedShadow,
"_msor");
1459 materializeOneCheck(IRB, Shadow,
nullptr);
1463 void materializeChecks() {
1465 [](
const ShadowOriginAndInsertPoint &L,
1466 const ShadowOriginAndInsertPoint &R) {
1467 return L.OrigIns <
R.OrigIns;
1470 for (
auto I = InstrumentationList.begin();
1471 I != InstrumentationList.end();) {
1473 std::find_if(
I + 1, InstrumentationList.end(),
1474 [L =
I->OrigIns](
const ShadowOriginAndInsertPoint &R) {
1475 return L != R.OrigIns;
1489 MS.ParamTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1490 {Zero, IRB.getInt32(0)},
"param_shadow");
1491 MS.RetvalTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1492 {Zero, IRB.getInt32(1)},
"retval_shadow");
1493 MS.VAArgTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1494 {Zero, IRB.getInt32(2)},
"va_arg_shadow");
1495 MS.VAArgOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1496 {Zero, IRB.getInt32(3)},
"va_arg_origin");
1497 MS.VAArgOverflowSizeTLS =
1498 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1499 {Zero, IRB.getInt32(4)},
"va_arg_overflow_size");
1500 MS.ParamOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1501 {Zero, IRB.getInt32(5)},
"param_origin");
1502 MS.RetvalOriginTLS =
1503 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1504 {Zero, IRB.getInt32(6)},
"retval_origin");
1506 MS.MsanMetadataAlloca = IRB.
CreateAlloca(MS.MsanMetadata, 0u);
1518 for (
PHINode *PN : ShadowPHINodes) {
1519 PHINode *PNS = cast<PHINode>(getShadow(PN));
1520 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1521 size_t NumValues = PN->getNumIncomingValues();
1522 for (
size_t v = 0;
v < NumValues;
v++) {
1523 PNS->
addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1525 PNO->
addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1529 VAHelper->finalizeInstrumentation();
1533 if (InstrumentLifetimeStart) {
1534 for (
auto Item : LifetimeStartList) {
1535 instrumentAlloca(*Item.second, Item.first);
1536 AllocaSet.
remove(Item.second);
1542 instrumentAlloca(*AI);
1545 materializeChecks();
1549 materializeStores();
1555 Type *getShadowTy(
Value *V) {
return getShadowTy(
V->getType()); }
1567 if (
VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1568 uint32_t EltSize =
DL.getTypeSizeInBits(VT->getElementType());
1570 VT->getElementCount());
1572 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1573 return ArrayType::get(getShadowTy(AT->getElementType()),
1574 AT->getNumElements());
1576 if (
StructType *ST = dyn_cast<StructType>(OrigTy)) {
1578 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1579 Elements.push_back(getShadowTy(
ST->getElementType(i)));
1581 LLVM_DEBUG(
dbgs() <<
"getShadowTy: " << *ST <<
" ===> " << *Res <<
"\n");
1597 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1599 if (Aggregator != FalseVal)
1600 Aggregator = IRB.
CreateOr(Aggregator, ShadowBool);
1602 Aggregator = ShadowBool;
1611 if (!
Array->getNumElements())
1615 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1619 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1620 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
1630 return collapseStructShadow(
Struct, V, IRB);
1631 if (
ArrayType *Array = dyn_cast<ArrayType>(
V->getType()))
1632 return collapseArrayShadow(Array, V, IRB);
1633 if (isa<VectorType>(
V->getType())) {
1634 if (isa<ScalableVectorType>(
V->getType()))
1637 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1645 Type *VTy =
V->getType();
1647 return convertToBool(convertShadowToScalar(V, IRB), IRB,
name);
1654 Type *ptrToIntPtrType(
Type *PtrTy)
const {
1655 if (
VectorType *VectTy = dyn_cast<VectorType>(PtrTy)) {
1656 return VectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1657 VectTy->getElementCount());
1663 Type *getPtrToShadowPtrType(
Type *IntPtrTy,
Type *ShadowTy)
const {
1664 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1665 return VectorType::get(
1666 getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1667 VectTy->getElementCount());
1669 assert(IntPtrTy == MS.IntptrTy);
1670 return PointerType::get(*MS.C, 0);
1674 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1676 VectTy->getElementCount(), constToIntPtr(VectTy->getElementType(),
C));
1678 assert(IntPtrTy == MS.IntptrTy);
1679 return ConstantInt::get(MS.IntptrTy,
C);
1690 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1693 if (
uint64_t AndMask = MS.MapParams->AndMask)
1694 OffsetLong = IRB.
CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1696 if (
uint64_t XorMask = MS.MapParams->XorMask)
1697 OffsetLong = IRB.
CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1709 std::pair<Value *, Value *>
1716 assert(VectTy->getElementType()->isPointerTy());
1718 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1719 Value *ShadowOffset = getShadowPtrOffset(
Addr, IRB);
1720 Value *ShadowLong = ShadowOffset;
1721 if (
uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1723 IRB.
CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1726 ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1728 Value *OriginPtr =
nullptr;
1729 if (MS.TrackOrigins) {
1730 Value *OriginLong = ShadowOffset;
1731 uint64_t OriginBase = MS.MapParams->OriginBase;
1732 if (OriginBase != 0)
1734 IRB.
CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1737 OriginLong = IRB.
CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1740 OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
1742 return std::make_pair(ShadowPtr, OriginPtr);
1745 template <
typename... ArgsTy>
1750 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1751 return IRB.
CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1754 return IRB.
CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1757 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(
Value *
Addr,
1761 Value *ShadowOriginPtrs;
1769 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1771 Value *SizeVal = ConstantInt::get(MS.IntptrTy,
Size);
1772 ShadowOriginPtrs = createMetadataCall(
1774 isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1781 return std::make_pair(ShadowPtr, OriginPtr);
1787 std::pair<Value *, Value *> getShadowOriginPtrKernel(
Value *
Addr,
1794 return getShadowOriginPtrKernelNoVec(
Addr, IRB, ShadowTy,
isStore);
1798 unsigned NumElements = cast<FixedVectorType>(VectTy)->getNumElements();
1799 Value *ShadowPtrs = ConstantInt::getNullValue(
1801 Value *OriginPtrs =
nullptr;
1802 if (MS.TrackOrigins)
1803 OriginPtrs = ConstantInt::getNullValue(
1805 for (
unsigned i = 0; i < NumElements; ++i) {
1808 auto [ShadowPtr, OriginPtr] =
1809 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy,
isStore);
1812 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1813 if (MS.TrackOrigins)
1815 OriginPtrs, OriginPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1817 return {ShadowPtrs, OriginPtrs};
1824 if (MS.CompileKernel)
1825 return getShadowOriginPtrKernel(
Addr, IRB, ShadowTy,
isStore);
1826 return getShadowOriginPtrUserspace(
Addr, IRB, ShadowTy, Alignment);
1841 if (!MS.TrackOrigins)
1855 Value *getOriginPtrForRetval() {
1857 return MS.RetvalOriginTLS;
1862 assert(!ShadowMap.
count(V) &&
"Values may only have one shadow");
1863 ShadowMap[
V] = PropagateShadow ? SV : getCleanShadow(V);
1868 if (!MS.TrackOrigins)
1870 assert(!OriginMap.
count(V) &&
"Values may only have one origin");
1871 LLVM_DEBUG(
dbgs() <<
"ORIGIN: " << *V <<
" ==> " << *Origin <<
"\n");
1872 OriginMap[
V] = Origin;
1876 Type *ShadowTy = getShadowTy(OrigTy);
1886 Constant *getCleanShadow(
Value *V) {
return getCleanShadow(
V->getType()); }
1891 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1893 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1895 getPoisonedShadow(AT->getElementType()));
1898 if (
StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1900 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1901 Vals.
push_back(getPoisonedShadow(
ST->getElementType(i)));
1909 Type *ShadowTy = getShadowTy(V);
1912 return getPoisonedShadow(ShadowTy);
1924 if (!PropagateShadow ||
I->getMetadata(LLVMContext::MD_nosanitize))
1925 return getCleanShadow(V);
1927 Value *Shadow = ShadowMap[
V];
1929 LLVM_DEBUG(
dbgs() <<
"No shadow: " << *V <<
"\n" << *(
I->getParent()));
1931 assert(Shadow &&
"No shadow for a value");
1935 if (
UndefValue *U = dyn_cast<UndefValue>(V)) {
1936 Value *
AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1937 : getCleanShadow(V);
1942 if (
Argument *
A = dyn_cast<Argument>(V)) {
1944 Value *&ShadowPtr = ShadowMap[
V];
1949 unsigned ArgOffset = 0;
1951 for (
auto &FArg :
F->args()) {
1952 if (!FArg.getType()->isSized()) {
1957 unsigned Size = FArg.hasByValAttr()
1958 ?
DL.getTypeAllocSize(FArg.getParamByValType())
1959 :
DL.getTypeAllocSize(FArg.getType());
1963 if (FArg.hasByValAttr()) {
1967 const Align ArgAlign =
DL.getValueOrABITypeAlignment(
1968 FArg.getParamAlign(), FArg.getParamByValType());
1969 Value *CpShadowPtr, *CpOriginPtr;
1970 std::tie(CpShadowPtr, CpOriginPtr) =
1971 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1973 if (!PropagateShadow || Overflow) {
1975 EntryIRB.CreateMemSet(
1979 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
1981 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign,
Base,
1986 if (MS.TrackOrigins) {
1988 getOriginPtrForArgument(EntryIRB, ArgOffset);
1992 EntryIRB.CreateMemCpy(
2001 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2002 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
2003 ShadowPtr = getCleanShadow(V);
2004 setOrigin(
A, getCleanOrigin());
2007 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2008 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg),
Base,
2010 if (MS.TrackOrigins) {
2012 getOriginPtrForArgument(EntryIRB, ArgOffset);
2013 setOrigin(
A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
2017 <<
" ARG: " << FArg <<
" ==> " << *ShadowPtr <<
"\n");
2023 assert(ShadowPtr &&
"Could not find shadow for an argument");
2027 return getCleanShadow(V);
2032 return getShadow(
I->getOperand(i));
2037 if (!MS.TrackOrigins)
2039 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
2040 return getCleanOrigin();
2041 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2042 "Unexpected value type in getOrigin()");
2044 if (
I->getMetadata(LLVMContext::MD_nosanitize))
2045 return getCleanOrigin();
2047 Value *Origin = OriginMap[
V];
2048 assert(Origin &&
"Missing origin");
2054 return getOrigin(
I->getOperand(i));
2067 LLVM_DEBUG(
dbgs() <<
"Skipping check of " << *Shadow <<
" before "
2068 << *OrigIns <<
"\n");
2073 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2074 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2075 "Can only insert checks for integer, vector, and aggregate shadow "
2078 InstrumentationList.push_back(
2079 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2088 Value *Shadow, *Origin;
2090 Shadow = getShadow(Val);
2093 Origin = getOrigin(Val);
2095 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2098 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
2100 insertShadowCheck(Shadow, Origin, OrigIns);
2105 case AtomicOrdering::NotAtomic:
2106 return AtomicOrdering::NotAtomic;
2107 case AtomicOrdering::Unordered:
2108 case AtomicOrdering::Monotonic:
2109 case AtomicOrdering::Release:
2110 return AtomicOrdering::Release;
2111 case AtomicOrdering::Acquire:
2112 case AtomicOrdering::AcquireRelease:
2113 return AtomicOrdering::AcquireRelease;
2114 case AtomicOrdering::SequentiallyConsistent:
2115 return AtomicOrdering::SequentiallyConsistent;
2121 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2122 uint32_t OrderingTable[NumOrderings] = {};
2124 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2125 OrderingTable[(
int)AtomicOrderingCABI::release] =
2126 (int)AtomicOrderingCABI::release;
2127 OrderingTable[(int)AtomicOrderingCABI::consume] =
2128 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2129 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2130 (
int)AtomicOrderingCABI::acq_rel;
2131 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2132 (
int)AtomicOrderingCABI::seq_cst;
2135 ArrayRef(OrderingTable, NumOrderings));
2140 case AtomicOrdering::NotAtomic:
2141 return AtomicOrdering::NotAtomic;
2142 case AtomicOrdering::Unordered:
2143 case AtomicOrdering::Monotonic:
2144 case AtomicOrdering::Acquire:
2145 return AtomicOrdering::Acquire;
2146 case AtomicOrdering::Release:
2147 case AtomicOrdering::AcquireRelease:
2148 return AtomicOrdering::AcquireRelease;
2149 case AtomicOrdering::SequentiallyConsistent:
2150 return AtomicOrdering::SequentiallyConsistent;
2156 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2157 uint32_t OrderingTable[NumOrderings] = {};
2159 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2160 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2161 OrderingTable[(int)AtomicOrderingCABI::consume] =
2162 (
int)AtomicOrderingCABI::acquire;
2163 OrderingTable[(int)AtomicOrderingCABI::release] =
2164 OrderingTable[(
int)AtomicOrderingCABI::acq_rel] =
2165 (int)AtomicOrderingCABI::acq_rel;
2166 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2167 (
int)AtomicOrderingCABI::seq_cst;
2170 ArrayRef(OrderingTable, NumOrderings));
2176 if (
I.getMetadata(LLVMContext::MD_nosanitize))
2179 if (isInPrologue(
I))
2189 assert(
I.getType()->isSized() &&
"Load type must have size");
2190 assert(!
I.getMetadata(LLVMContext::MD_nosanitize));
2191 NextNodeIRBuilder IRB(&
I);
2192 Type *ShadowTy = getShadowTy(&
I);
2194 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2195 const Align Alignment =
I.getAlign();
2196 if (PropagateShadow) {
2197 std::tie(ShadowPtr, OriginPtr) =
2198 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2202 setShadow(&
I, getCleanShadow(&
I));
2206 insertShadowCheck(
I.getPointerOperand(), &
I);
2211 if (MS.TrackOrigins) {
2212 if (PropagateShadow) {
2217 setOrigin(&
I, getCleanOrigin());
2227 StoreList.push_back(&
I);
2229 insertShadowCheck(
I.getPointerOperand(), &
I);
2233 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2237 Value *Val =
I.getOperand(1);
2238 Value *ShadowPtr = getShadowOriginPtr(
Addr, IRB, getShadowTy(Val),
Align(1),
2243 insertShadowCheck(
Addr, &
I);
2248 if (isa<AtomicCmpXchgInst>(
I))
2249 insertShadowCheck(Val, &
I);
2253 setShadow(&
I, getCleanShadow(&
I));
2254 setOrigin(&
I, getCleanOrigin());
2269 insertShadowCheck(
I.getOperand(1), &
I);
2273 setOrigin(&
I, getOrigin(&
I, 0));
2277 insertShadowCheck(
I.getOperand(2), &
I);
2279 auto *Shadow0 = getShadow(&
I, 0);
2280 auto *Shadow1 = getShadow(&
I, 1);
2283 setOriginForNaryOp(
I);
2288 auto *Shadow0 = getShadow(&
I, 0);
2289 auto *Shadow1 = getShadow(&
I, 1);
2292 setOriginForNaryOp(
I);
2298 setShadow(&
I, IRB.
CreateSExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2299 setOrigin(&
I, getOrigin(&
I, 0));
2304 setShadow(&
I, IRB.
CreateZExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2305 setOrigin(&
I, getOrigin(&
I, 0));
2310 setShadow(&
I, IRB.
CreateTrunc(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2311 setOrigin(&
I, getOrigin(&
I, 0));
2318 if (
auto *CI = dyn_cast<CallInst>(
I.getOperand(0)))
2319 if (CI->isMustTailCall())
2323 setOrigin(&
I, getOrigin(&
I, 0));
2329 "_msprop_ptrtoint"));
2330 setOrigin(&
I, getOrigin(&
I, 0));
2336 "_msprop_inttoptr"));
2337 setOrigin(&
I, getOrigin(&
I, 0));
2340 void visitFPToSIInst(
CastInst &
I) { handleShadowOr(
I); }
2341 void visitFPToUIInst(
CastInst &
I) { handleShadowOr(
I); }
2342 void visitSIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2343 void visitUIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2344 void visitFPExtInst(
CastInst &
I) { handleShadowOr(
I); }
2345 void visitFPTruncInst(
CastInst &
I) { handleShadowOr(
I); }
2360 Value *S2 = getShadow(&
I, 1);
2361 Value *V1 =
I.getOperand(0);
2370 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2371 setOriginForNaryOp(
I);
2382 Value *S2 = getShadow(&
I, 1);
2392 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2393 setOriginForNaryOp(
I);
2411 template <
bool CombineShadow>
class Combiner {
2412 Value *Shadow =
nullptr;
2413 Value *Origin =
nullptr;
2415 MemorySanitizerVisitor *MSV;
2419 : IRB(IRB), MSV(MSV) {}
2423 if (CombineShadow) {
2428 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2429 Shadow = IRB.
CreateOr(Shadow, OpShadow,
"_msprop");
2433 if (MSV->MS.TrackOrigins) {
2438 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2440 if (!ConstOrigin || !ConstOrigin->
isNullValue()) {
2441 Value *
Cond = MSV->convertToBool(OpShadow, IRB);
2451 Value *OpShadow = MSV->getShadow(V);
2452 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) :
nullptr;
2453 return Add(OpShadow, OpOrigin);
2459 if (CombineShadow) {
2461 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(
I));
2462 MSV->setShadow(
I, Shadow);
2464 if (MSV->MS.TrackOrigins) {
2466 MSV->setOrigin(
I, Origin);
2476 if (!MS.TrackOrigins)
2479 OriginCombiner
OC(
this, IRB);
2480 for (
Use &
Op :
I.operands())
2485 size_t VectorOrPrimitiveTypeSizeInBits(
Type *Ty) {
2487 "Vector of pointers is not a valid shadow type");
2488 return Ty->
isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2497 Type *srcTy =
V->getType();
2498 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2499 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2500 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2506 cast<VectorType>(dstTy)->getElementCount() ==
2507 cast<VectorType>(srcTy)->getElementCount())
2518 Type *ShadowTy = getShadowTy(V);
2519 if (
V->getType() == ShadowTy)
2521 if (
V->getType()->isPtrOrPtrVectorTy())
2530 ShadowAndOriginCombiner
SC(
this, IRB);
2531 for (
Use &
Op :
I.operands())
2551 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
2552 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2553 Type *EltTy = VTy->getElementType();
2555 for (
unsigned Idx = 0;
Idx < NumElements; ++
Idx) {
2558 const APInt &
V = Elt->getValue();
2560 Elements.push_back(ConstantInt::get(EltTy, V2));
2562 Elements.push_back(ConstantInt::get(EltTy, 1));
2567 if (
ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2568 const APInt &
V = Elt->getValue();
2570 ShadowMul = ConstantInt::get(Ty, V2);
2572 ShadowMul = ConstantInt::get(Ty, 1);
2578 IRB.
CreateMul(getShadow(OtherArg), ShadowMul,
"msprop_mul_cst"));
2579 setOrigin(&
I, getOrigin(OtherArg));
2583 Constant *constOp0 = dyn_cast<Constant>(
I.getOperand(0));
2584 Constant *constOp1 = dyn_cast<Constant>(
I.getOperand(1));
2585 if (constOp0 && !constOp1)
2586 handleMulByConstant(
I, constOp0,
I.getOperand(1));
2587 else if (constOp1 && !constOp0)
2588 handleMulByConstant(
I, constOp1,
I.getOperand(0));
2603 insertShadowCheck(
I.getOperand(1), &
I);
2604 setShadow(&
I, getShadow(&
I, 0));
2605 setOrigin(&
I, getOrigin(&
I, 0));
2622 void handleEqualityComparison(
ICmpInst &
I) {
2626 Value *Sa = getShadow(
A);
2627 Value *Sb = getShadow(
B);
2653 setOriginForNaryOp(
I);
2695 void handleRelationalComparisonExact(
ICmpInst &
I) {
2699 Value *Sa = getShadow(
A);
2700 Value *Sb = getShadow(
B);
2711 bool IsSigned =
I.isSigned();
2713 getLowestPossibleValue(IRB,
A, Sa, IsSigned),
2714 getHighestPossibleValue(IRB,
B, Sb, IsSigned));
2716 getHighestPossibleValue(IRB,
A, Sa, IsSigned),
2717 getLowestPossibleValue(IRB,
B, Sb, IsSigned));
2720 setOriginForNaryOp(
I);
2727 void handleSignedRelationalComparison(
ICmpInst &
I) {
2731 if ((constOp = dyn_cast<Constant>(
I.getOperand(1)))) {
2732 op =
I.getOperand(0);
2733 pre =
I.getPredicate();
2734 }
else if ((constOp = dyn_cast<Constant>(
I.getOperand(0)))) {
2735 op =
I.getOperand(1);
2736 pre =
I.getSwappedPredicate();
2749 setShadow(&
I, Shadow);
2750 setOrigin(&
I, getOrigin(
op));
2761 if (
I.isEquality()) {
2762 handleEqualityComparison(
I);
2768 handleRelationalComparisonExact(
I);
2772 handleSignedRelationalComparison(
I);
2777 if ((isa<Constant>(
I.getOperand(0)) || isa<Constant>(
I.getOperand(1)))) {
2778 handleRelationalComparisonExact(
I);
2785 void visitFCmpInst(
FCmpInst &
I) { handleShadowOr(
I); }
2792 Value *S2 = getShadow(&
I, 1);
2797 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2798 setOriginForNaryOp(
I);
2809 Value *S0 = getShadow(&
I, 0);
2811 Value *S2 = getShadow(&
I, 2);
2816 I.getModule(),
I.getIntrinsicID(), S2Conv->
getType());
2818 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2819 setOriginForNaryOp(
I);
2833 getShadow(
I.getArgOperand(1));
2836 {I.getArgOperand(0), I.getArgOperand(1),
2837 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2838 I.eraseFromParent();
2856 getShadow(
I.getArgOperand(1));
2859 {I.getArgOperand(0), I.getArgOperand(1),
2860 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2861 I.eraseFromParent();
2869 {I.getArgOperand(0),
2870 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2871 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2872 I.eraseFromParent();
2875 void visitVAStartInst(
VAStartInst &
I) { VAHelper->visitVAStartInst(
I); }
2877 void visitVACopyInst(
VACopyInst &
I) { VAHelper->visitVACopyInst(
I); }
2886 Value *Shadow = getShadow(&
I, 1);
2887 Value *ShadowPtr, *OriginPtr;
2891 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2896 insertShadowCheck(
Addr, &
I);
2899 if (MS.TrackOrigins)
2912 Type *ShadowTy = getShadowTy(&
I);
2913 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2914 if (PropagateShadow) {
2918 std::tie(ShadowPtr, OriginPtr) =
2919 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2923 setShadow(&
I, getCleanShadow(&
I));
2927 insertShadowCheck(
Addr, &
I);
2929 if (MS.TrackOrigins) {
2930 if (PropagateShadow)
2931 setOrigin(&
I, IRB.
CreateLoad(MS.OriginTy, OriginPtr));
2933 setOrigin(&
I, getCleanOrigin());
2946 if (!(
RetTy->isIntOrIntVectorTy() ||
RetTy->isFPOrFPVectorTy() ||
2947 RetTy->isX86_MMXTy()))
2950 unsigned NumArgOperands =
I.arg_size();
2951 for (
unsigned i = 0; i < NumArgOperands; ++i) {
2952 Type *Ty =
I.getArgOperand(i)->getType();
2958 ShadowAndOriginCombiner
SC(
this, IRB);
2959 for (
unsigned i = 0; i < NumArgOperands; ++i)
2960 SC.Add(
I.getArgOperand(i));
2977 unsigned NumArgOperands =
I.arg_size();
2978 if (NumArgOperands == 0)
2981 if (NumArgOperands == 2 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
2982 I.getArgOperand(1)->getType()->isVectorTy() &&
2983 I.getType()->isVoidTy() && !
I.onlyReadsMemory()) {
2985 return handleVectorStoreIntrinsic(
I);
2988 if (NumArgOperands == 1 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
2989 I.getType()->isVectorTy() &&
I.onlyReadsMemory()) {
2991 return handleVectorLoadIntrinsic(
I);
2994 if (
I.doesNotAccessMemory())
2995 if (maybeHandleSimpleNomemIntrinsic(
I))
3003 setShadow(&
I, getShadow(&
I, 0));
3004 setOrigin(&
I, getOrigin(&
I, 0));
3012 InstrumentLifetimeStart =
false;
3013 LifetimeStartList.push_back(std::make_pair(&
I, AI));
3019 Type *OpType =
Op->getType();
3021 F.getParent(), Intrinsic::bswap,
ArrayRef(&OpType, 1));
3023 setOrigin(&
I, getOrigin(
Op));
3028 Value *Src =
I.getArgOperand(0);
3034 Constant *IsZeroPoison = cast<Constant>(
I.getOperand(1));
3037 BoolShadow = IRB.
CreateOr(BoolShadow, BoolZeroPoison,
"_mscz_bs");
3040 Value *OutputShadow =
3041 IRB.
CreateSExt(BoolShadow, getShadowTy(Src),
"_mscz_os");
3043 setShadow(&
I, OutputShadow);
3044 setOriginForNaryOp(
I);
3062 void handleVectorConvertIntrinsic(
IntrinsicInst &
I,
int NumUsedElements,
3063 bool HasRoundingMode =
false) {
3065 Value *CopyOp, *ConvertOp;
3067 assert((!HasRoundingMode ||
3068 isa<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))) &&
3069 "Invalid rounding mode");
3071 switch (
I.arg_size() - HasRoundingMode) {
3073 CopyOp =
I.getArgOperand(0);
3074 ConvertOp =
I.getArgOperand(1);
3077 ConvertOp =
I.getArgOperand(0);
3091 Value *ConvertShadow = getShadow(ConvertOp);
3092 Value *AggShadow =
nullptr;
3095 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), 0));
3096 for (
int i = 1; i < NumUsedElements; ++i) {
3098 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), i));
3099 AggShadow = IRB.
CreateOr(AggShadow, MoreShadow);
3102 AggShadow = ConvertShadow;
3105 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &
I);
3112 Value *ResultShadow = getShadow(CopyOp);
3113 Type *EltTy = cast<VectorType>(ResultShadow->
getType())->getElementType();
3114 for (
int i = 0; i < NumUsedElements; ++i) {
3116 ResultShadow, ConstantInt::getNullValue(EltTy),
3119 setShadow(&
I, ResultShadow);
3120 setOrigin(&
I, getOrigin(CopyOp));
3122 setShadow(&
I, getCleanShadow(&
I));
3123 setOrigin(&
I, getCleanOrigin());
3131 S = CreateShadowCast(IRB, S, IRB.
getInt64Ty(),
true);
3134 return CreateShadowCast(IRB, S2,
T,
true);
3142 return CreateShadowCast(IRB, S2,
T,
true);
3159 void handleVectorShiftIntrinsic(
IntrinsicInst &
I,
bool Variable) {
3165 Value *S2 = getShadow(&
I, 1);
3166 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3167 : Lower64ShadowExtend(IRB, S2, getShadowTy(&
I));
3168 Value *V1 =
I.getOperand(0);
3171 {IRB.CreateBitCast(S1, V1->getType()), V2});
3173 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
3174 setOriginForNaryOp(
I);
3178 Type *getMMXVectorTy(
unsigned EltSizeInBits) {
3179 const unsigned X86_MMXSizeInBits = 64;
3180 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3181 "Illegal MMX vector element size");
3183 X86_MMXSizeInBits / EltSizeInBits);
3190 case Intrinsic::x86_sse2_packsswb_128:
3191 case Intrinsic::x86_sse2_packuswb_128:
3192 return Intrinsic::x86_sse2_packsswb_128;
3194 case Intrinsic::x86_sse2_packssdw_128:
3195 case Intrinsic::x86_sse41_packusdw:
3196 return Intrinsic::x86_sse2_packssdw_128;
3198 case Intrinsic::x86_avx2_packsswb:
3199 case Intrinsic::x86_avx2_packuswb:
3200 return Intrinsic::x86_avx2_packsswb;
3202 case Intrinsic::x86_avx2_packssdw:
3203 case Intrinsic::x86_avx2_packusdw:
3204 return Intrinsic::x86_avx2_packssdw;
3206 case Intrinsic::x86_mmx_packsswb:
3207 case Intrinsic::x86_mmx_packuswb:
3208 return Intrinsic::x86_mmx_packsswb;
3210 case Intrinsic::x86_mmx_packssdw:
3211 return Intrinsic::x86_mmx_packssdw;
3224 void handleVectorPackIntrinsic(
IntrinsicInst &
I,
unsigned EltSizeInBits = 0) {
3226 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3229 Value *S2 = getShadow(&
I, 1);
3230 assert(isX86_MMX ||
S1->getType()->isVectorTy());
3235 Type *
T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) :
S1->
getType();
3251 F.getParent(), getSignedPackIntrinsic(
I.getIntrinsicID()));
3254 IRB.
CreateCall(ShadowFn, {S1_ext, S2_ext},
"_msprop_vector_pack");
3258 setOriginForNaryOp(
I);
3263 const unsigned SignificantBitsPerResultElement = 16;
3264 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3266 unsigned ZeroBitsPerResultElement =
3270 auto *Shadow0 = getShadow(&
I, 0);
3271 auto *Shadow1 = getShadow(&
I, 1);
3276 S = IRB.
CreateLShr(S, ZeroBitsPerResultElement);
3279 setOriginForNaryOp(
I);
3284 unsigned EltSizeInBits = 0) {
3285 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3286 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) :
I.
getType();
3288 auto *Shadow0 = getShadow(&
I, 0);
3289 auto *Shadow1 = getShadow(&
I, 1);
3296 setOriginForNaryOp(
I);
3304 Type *ResTy = getShadowTy(&
I);
3305 auto *Shadow0 = getShadow(&
I, 0);
3306 auto *Shadow1 = getShadow(&
I, 1);
3311 setOriginForNaryOp(
I);
3319 auto *Shadow0 = getShadow(&
I, 0);
3320 auto *Shadow1 = getShadow(&
I, 1);
3322 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&
I));
3324 setOriginForNaryOp(
I);
3333 setOrigin(&
I, getOrigin(&
I, 0));
3341 Value *OperandShadow = getShadow(&
I, 0);
3343 Value *OperandUnsetOrPoison = IRB.
CreateOr(OperandUnsetBits, OperandShadow);
3351 setOrigin(&
I, getOrigin(&
I, 0));
3359 Value *OperandShadow = getShadow(&
I, 0);
3360 Value *OperandSetOrPoison = IRB.
CreateOr(
I.getOperand(0), OperandShadow);
3368 setOrigin(&
I, getOrigin(&
I, 0));
3376 getShadowOriginPtr(
Addr, IRB, Ty,
Align(1),
true).first;
3381 insertShadowCheck(
Addr, &
I);
3392 Value *ShadowPtr, *OriginPtr;
3393 std::tie(ShadowPtr, OriginPtr) =
3394 getShadowOriginPtr(
Addr, IRB, Ty, Alignment,
false);
3397 insertShadowCheck(
Addr, &
I);
3400 Value *Origin = MS.TrackOrigins ? IRB.
CreateLoad(MS.OriginTy, OriginPtr)
3402 insertShadowCheck(Shadow, Origin, &
I);
3409 Value *PassThru =
I.getArgOperand(2);
3412 insertShadowCheck(
Ptr, &
I);
3413 insertShadowCheck(Mask, &
I);
3416 if (!PropagateShadow) {
3417 setShadow(&
I, getCleanShadow(&
I));
3418 setOrigin(&
I, getCleanOrigin());
3422 Type *ShadowTy = getShadowTy(&
I);
3423 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3424 auto [ShadowPtr, OriginPtr] =
3425 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
false);
3428 ShadowTy, ShadowPtr, Mask, getShadow(PassThru),
"_msmaskedexpload");
3430 setShadow(&
I, Shadow);
3433 setOrigin(&
I, getCleanOrigin());
3438 Value *Values =
I.getArgOperand(0);
3443 insertShadowCheck(
Ptr, &
I);
3444 insertShadowCheck(Mask, &
I);
3447 Value *Shadow = getShadow(Values);
3448 Type *ElementShadowTy =
3449 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3450 auto [ShadowPtr, OriginPtrs] =
3451 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
true);
3460 Value *Ptrs =
I.getArgOperand(0);
3461 const Align Alignment(
3462 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3464 Value *PassThru =
I.getArgOperand(3);
3466 Type *PtrsShadowTy = getShadowTy(Ptrs);
3468 insertShadowCheck(Mask, &
I);
3472 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3475 if (!PropagateShadow) {
3476 setShadow(&
I, getCleanShadow(&
I));
3477 setOrigin(&
I, getCleanOrigin());
3481 Type *ShadowTy = getShadowTy(&
I);
3482 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3483 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3484 Ptrs, IRB, ElementShadowTy, Alignment,
false);
3488 getShadow(PassThru),
"_msmaskedgather");
3490 setShadow(&
I, Shadow);
3493 setOrigin(&
I, getCleanOrigin());
3498 Value *Values =
I.getArgOperand(0);
3499 Value *Ptrs =
I.getArgOperand(1);
3500 const Align Alignment(
3501 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3504 Type *PtrsShadowTy = getShadowTy(Ptrs);
3506 insertShadowCheck(Mask, &
I);
3510 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3513 Value *Shadow = getShadow(Values);
3514 Type *ElementShadowTy =
3515 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3516 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3517 Ptrs, IRB, ElementShadowTy, Alignment,
true);
3526 Value *
V =
I.getArgOperand(0);
3528 const Align Alignment(
3529 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3531 Value *Shadow = getShadow(V);
3534 insertShadowCheck(
Ptr, &
I);
3535 insertShadowCheck(Mask, &
I);
3540 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3541 Ptr, IRB, Shadow->
getType(), Alignment,
true);
3545 if (!MS.TrackOrigins)
3548 auto &
DL =
F.getParent()->getDataLayout();
3549 paintOrigin(IRB, getOrigin(V), OriginPtr,
3557 const Align Alignment(
3558 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3560 Value *PassThru =
I.getArgOperand(3);
3563 insertShadowCheck(
Ptr, &
I);
3564 insertShadowCheck(Mask, &
I);
3567 if (!PropagateShadow) {
3568 setShadow(&
I, getCleanShadow(&
I));
3569 setOrigin(&
I, getCleanOrigin());
3573 Type *ShadowTy = getShadowTy(&
I);
3574 Value *ShadowPtr, *OriginPtr;
3575 std::tie(ShadowPtr, OriginPtr) =
3576 getShadowOriginPtr(
Ptr, IRB, ShadowTy, Alignment,
false);
3578 getShadow(PassThru),
"_msmaskedld"));
3580 if (!MS.TrackOrigins)
3587 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB,
"_mscmp");
3592 setOrigin(&
I, Origin);
3602 Type *ShadowTy = getShadowTy(&
I);
3605 Value *SMask = getShadow(&
I, 1);
3610 {getShadow(&I, 0), I.getOperand(1)});
3613 setOriginForNaryOp(
I);
3618 for (
unsigned X = OddElements ? 1 : 0;
X < Width;
X += 2) {
3635 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3636 assert(isa<ConstantInt>(
I.getArgOperand(2)) &&
3637 "pclmul 3rd operand must be a constant");
3638 unsigned Imm = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3640 getPclmulMask(Width, Imm & 0x01));
3642 getPclmulMask(Width, Imm & 0x10));
3643 ShadowAndOriginCombiner SOC(
this, IRB);
3644 SOC.Add(Shuf0, getOrigin(&
I, 0));
3645 SOC.Add(Shuf1, getOrigin(&
I, 1));
3653 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3655 Value *Second = getShadow(&
I, 1);
3658 Mask.push_back(Width);
3659 for (
unsigned i = 1; i < Width; i++)
3663 setShadow(&
I, Shadow);
3664 setOriginForNaryOp(
I);
3669 Value *Shadow0 = getShadow(&
I, 0);
3670 Value *Shadow1 = getShadow(&
I, 1);
3676 setShadow(&
I, Shadow);
3677 setOriginForNaryOp(
I);
3683 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3685 Value *Second = getShadow(&
I, 1);
3689 Mask.push_back(Width);
3690 for (
unsigned i = 1; i < Width; i++)
3694 setShadow(&
I, Shadow);
3695 setOriginForNaryOp(
I);
3702 assert(
I.getType()->isIntOrIntVectorTy());
3703 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3707 setShadow(&
I, getShadow(&
I, 0));
3708 setOrigin(&
I, getOrigin(&
I, 0));
3713 Value *Shadow = getShadow(&
I, 0);
3714 setShadow(&
I, IRB.
CreateICmpNE(Shadow, getCleanShadow(Shadow)));
3715 setOrigin(&
I, getOrigin(&
I, 0));
3719 switch (
I.getIntrinsicID()) {
3720 case Intrinsic::abs:
3721 handleAbsIntrinsic(
I);
3723 case Intrinsic::is_fpclass:
3726 case Intrinsic::lifetime_start:
3727 handleLifetimeStart(
I);
3729 case Intrinsic::launder_invariant_group:
3730 case Intrinsic::strip_invariant_group:
3731 handleInvariantGroup(
I);
3733 case Intrinsic::bswap:
3736 case Intrinsic::ctlz:
3737 case Intrinsic::cttz:
3738 handleCountZeroes(
I);
3740 case Intrinsic::masked_compressstore:
3741 handleMaskedCompressStore(
I);
3743 case Intrinsic::masked_expandload:
3744 handleMaskedExpandLoad(
I);
3746 case Intrinsic::masked_gather:
3747 handleMaskedGather(
I);
3749 case Intrinsic::masked_scatter:
3750 handleMaskedScatter(
I);
3752 case Intrinsic::masked_store:
3753 handleMaskedStore(
I);
3755 case Intrinsic::masked_load:
3756 handleMaskedLoad(
I);
3758 case Intrinsic::vector_reduce_and:
3759 handleVectorReduceAndIntrinsic(
I);
3761 case Intrinsic::vector_reduce_or:
3762 handleVectorReduceOrIntrinsic(
I);
3764 case Intrinsic::vector_reduce_add:
3765 case Intrinsic::vector_reduce_xor:
3766 case Intrinsic::vector_reduce_mul:
3767 handleVectorReduceIntrinsic(
I);
3769 case Intrinsic::x86_sse_stmxcsr:
3772 case Intrinsic::x86_sse_ldmxcsr:
3775 case Intrinsic::x86_avx512_vcvtsd2usi64:
3776 case Intrinsic::x86_avx512_vcvtsd2usi32:
3777 case Intrinsic::x86_avx512_vcvtss2usi64:
3778 case Intrinsic::x86_avx512_vcvtss2usi32:
3779 case Intrinsic::x86_avx512_cvttss2usi64:
3780 case Intrinsic::x86_avx512_cvttss2usi:
3781 case Intrinsic::x86_avx512_cvttsd2usi64:
3782 case Intrinsic::x86_avx512_cvttsd2usi:
3783 case Intrinsic::x86_avx512_cvtusi2ss:
3784 case Intrinsic::x86_avx512_cvtusi642sd:
3785 case Intrinsic::x86_avx512_cvtusi642ss:
3786 handleVectorConvertIntrinsic(
I, 1,
true);
3788 case Intrinsic::x86_sse2_cvtsd2si64:
3789 case Intrinsic::x86_sse2_cvtsd2si:
3790 case Intrinsic::x86_sse2_cvtsd2ss:
3791 case Intrinsic::x86_sse2_cvttsd2si64:
3792 case Intrinsic::x86_sse2_cvttsd2si:
3793 case Intrinsic::x86_sse_cvtss2si64:
3794 case Intrinsic::x86_sse_cvtss2si:
3795 case Intrinsic::x86_sse_cvttss2si64:
3796 case Intrinsic::x86_sse_cvttss2si:
3797 handleVectorConvertIntrinsic(
I, 1);
3799 case Intrinsic::x86_sse_cvtps2pi:
3800 case Intrinsic::x86_sse_cvttps2pi:
3801 handleVectorConvertIntrinsic(
I, 2);
3804 case Intrinsic::x86_avx512_psll_w_512:
3805 case Intrinsic::x86_avx512_psll_d_512:
3806 case Intrinsic::x86_avx512_psll_q_512:
3807 case Intrinsic::x86_avx512_pslli_w_512:
3808 case Intrinsic::x86_avx512_pslli_d_512:
3809 case Intrinsic::x86_avx512_pslli_q_512:
3810 case Intrinsic::x86_avx512_psrl_w_512:
3811 case Intrinsic::x86_avx512_psrl_d_512:
3812 case Intrinsic::x86_avx512_psrl_q_512:
3813 case Intrinsic::x86_avx512_psra_w_512:
3814 case Intrinsic::x86_avx512_psra_d_512:
3815 case Intrinsic::x86_avx512_psra_q_512:
3816 case Intrinsic::x86_avx512_psrli_w_512:
3817 case Intrinsic::x86_avx512_psrli_d_512:
3818 case Intrinsic::x86_avx512_psrli_q_512:
3819 case Intrinsic::x86_avx512_psrai_w_512:
3820 case Intrinsic::x86_avx512_psrai_d_512:
3821 case Intrinsic::x86_avx512_psrai_q_512:
3822 case Intrinsic::x86_avx512_psra_q_256:
3823 case Intrinsic::x86_avx512_psra_q_128:
3824 case Intrinsic::x86_avx512_psrai_q_256:
3825 case Intrinsic::x86_avx512_psrai_q_128:
3826 case Intrinsic::x86_avx2_psll_w:
3827 case Intrinsic::x86_avx2_psll_d:
3828 case Intrinsic::x86_avx2_psll_q:
3829 case Intrinsic::x86_avx2_pslli_w:
3830 case Intrinsic::x86_avx2_pslli_d:
3831 case Intrinsic::x86_avx2_pslli_q:
3832 case Intrinsic::x86_avx2_psrl_w:
3833 case Intrinsic::x86_avx2_psrl_d:
3834 case Intrinsic::x86_avx2_psrl_q:
3835 case Intrinsic::x86_avx2_psra_w:
3836 case Intrinsic::x86_avx2_psra_d:
3837 case Intrinsic::x86_avx2_psrli_w:
3838 case Intrinsic::x86_avx2_psrli_d:
3839 case Intrinsic::x86_avx2_psrli_q:
3840 case Intrinsic::x86_avx2_psrai_w:
3841 case Intrinsic::x86_avx2_psrai_d:
3842 case Intrinsic::x86_sse2_psll_w:
3843 case Intrinsic::x86_sse2_psll_d:
3844 case Intrinsic::x86_sse2_psll_q:
3845 case Intrinsic::x86_sse2_pslli_w:
3846 case Intrinsic::x86_sse2_pslli_d:
3847 case Intrinsic::x86_sse2_pslli_q:
3848 case Intrinsic::x86_sse2_psrl_w:
3849 case Intrinsic::x86_sse2_psrl_d:
3850 case Intrinsic::x86_sse2_psrl_q:
3851 case Intrinsic::x86_sse2_psra_w:
3852 case Intrinsic::x86_sse2_psra_d:
3853 case Intrinsic::x86_sse2_psrli_w:
3854 case Intrinsic::x86_sse2_psrli_d:
3855 case Intrinsic::x86_sse2_psrli_q:
3856 case Intrinsic::x86_sse2_psrai_w:
3857 case Intrinsic::x86_sse2_psrai_d:
3858 case Intrinsic::x86_mmx_psll_w:
3859 case Intrinsic::x86_mmx_psll_d:
3860 case Intrinsic::x86_mmx_psll_q:
3861 case Intrinsic::x86_mmx_pslli_w:
3862 case Intrinsic::x86_mmx_pslli_d:
3863 case Intrinsic::x86_mmx_pslli_q:
3864 case Intrinsic::x86_mmx_psrl_w:
3865 case Intrinsic::x86_mmx_psrl_d:
3866 case Intrinsic::x86_mmx_psrl_q:
3867 case Intrinsic::x86_mmx_psra_w:
3868 case Intrinsic::x86_mmx_psra_d:
3869 case Intrinsic::x86_mmx_psrli_w:
3870 case Intrinsic::x86_mmx_psrli_d:
3871 case Intrinsic::x86_mmx_psrli_q:
3872 case Intrinsic::x86_mmx_psrai_w:
3873 case Intrinsic::x86_mmx_psrai_d:
3874 handleVectorShiftIntrinsic(
I,
false);
3876 case Intrinsic::x86_avx2_psllv_d:
3877 case Intrinsic::x86_avx2_psllv_d_256:
3878 case Intrinsic::x86_avx512_psllv_d_512:
3879 case Intrinsic::x86_avx2_psllv_q:
3880 case Intrinsic::x86_avx2_psllv_q_256:
3881 case Intrinsic::x86_avx512_psllv_q_512:
3882 case Intrinsic::x86_avx2_psrlv_d:
3883 case Intrinsic::x86_avx2_psrlv_d_256:
3884 case Intrinsic::x86_avx512_psrlv_d_512:
3885 case Intrinsic::x86_avx2_psrlv_q:
3886 case Intrinsic::x86_avx2_psrlv_q_256:
3887 case Intrinsic::x86_avx512_psrlv_q_512:
3888 case Intrinsic::x86_avx2_psrav_d:
3889 case Intrinsic::x86_avx2_psrav_d_256:
3890 case Intrinsic::x86_avx512_psrav_d_512:
3891 case Intrinsic::x86_avx512_psrav_q_128:
3892 case Intrinsic::x86_avx512_psrav_q_256:
3893 case Intrinsic::x86_avx512_psrav_q_512:
3894 handleVectorShiftIntrinsic(
I,
true);
3897 case Intrinsic::x86_sse2_packsswb_128:
3898 case Intrinsic::x86_sse2_packssdw_128:
3899 case Intrinsic::x86_sse2_packuswb_128:
3900 case Intrinsic::x86_sse41_packusdw:
3901 case Intrinsic::x86_avx2_packsswb:
3902 case Intrinsic::x86_avx2_packssdw:
3903 case Intrinsic::x86_avx2_packuswb:
3904 case Intrinsic::x86_avx2_packusdw:
3905 handleVectorPackIntrinsic(
I);
3908 case Intrinsic::x86_mmx_packsswb:
3909 case Intrinsic::x86_mmx_packuswb:
3910 handleVectorPackIntrinsic(
I, 16);
3913 case Intrinsic::x86_mmx_packssdw:
3914 handleVectorPackIntrinsic(
I, 32);
3917 case Intrinsic::x86_mmx_psad_bw:
3918 case Intrinsic::x86_sse2_psad_bw:
3919 case Intrinsic::x86_avx2_psad_bw:
3920 handleVectorSadIntrinsic(
I);
3923 case Intrinsic::x86_sse2_pmadd_wd:
3924 case Intrinsic::x86_avx2_pmadd_wd:
3925 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3926 case Intrinsic::x86_avx2_pmadd_ub_sw:
3927 handleVectorPmaddIntrinsic(
I);
3930 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3931 handleVectorPmaddIntrinsic(
I, 8);
3934 case Intrinsic::x86_mmx_pmadd_wd:
3935 handleVectorPmaddIntrinsic(
I, 16);
3938 case Intrinsic::x86_sse_cmp_ss:
3939 case Intrinsic::x86_sse2_cmp_sd:
3940 case Intrinsic::x86_sse_comieq_ss:
3941 case Intrinsic::x86_sse_comilt_ss:
3942 case Intrinsic::x86_sse_comile_ss:
3943 case Intrinsic::x86_sse_comigt_ss:
3944 case Intrinsic::x86_sse_comige_ss:
3945 case Intrinsic::x86_sse_comineq_ss:
3946 case Intrinsic::x86_sse_ucomieq_ss:
3947 case Intrinsic::x86_sse_ucomilt_ss:
3948 case Intrinsic::x86_sse_ucomile_ss:
3949 case Intrinsic::x86_sse_ucomigt_ss:
3950 case Intrinsic::x86_sse_ucomige_ss:
3951 case Intrinsic::x86_sse_ucomineq_ss:
3952 case Intrinsic::x86_sse2_comieq_sd:
3953 case Intrinsic::x86_sse2_comilt_sd:
3954 case Intrinsic::x86_sse2_comile_sd:
3955 case Intrinsic::x86_sse2_comigt_sd:
3956 case Intrinsic::x86_sse2_comige_sd:
3957 case Intrinsic::x86_sse2_comineq_sd:
3958 case Intrinsic::x86_sse2_ucomieq_sd:
3959 case Intrinsic::x86_sse2_ucomilt_sd:
3960 case Intrinsic::x86_sse2_ucomile_sd:
3961 case Intrinsic::x86_sse2_ucomigt_sd:
3962 case Intrinsic::x86_sse2_ucomige_sd:
3963 case Intrinsic::x86_sse2_ucomineq_sd:
3964 handleVectorCompareScalarIntrinsic(
I);
3967 case Intrinsic::x86_avx_cmp_pd_256:
3968 case Intrinsic::x86_avx_cmp_ps_256:
3969 case Intrinsic::x86_sse2_cmp_pd:
3970 case Intrinsic::x86_sse_cmp_ps:
3971 handleVectorComparePackedIntrinsic(
I);
3974 case Intrinsic::x86_bmi_bextr_32:
3975 case Intrinsic::x86_bmi_bextr_64:
3976 case Intrinsic::x86_bmi_bzhi_32:
3977 case Intrinsic::x86_bmi_bzhi_64:
3978 case Intrinsic::x86_bmi_pdep_32:
3979 case Intrinsic::x86_bmi_pdep_64:
3980 case Intrinsic::x86_bmi_pext_32:
3981 case Intrinsic::x86_bmi_pext_64:
3982 handleBmiIntrinsic(
I);
3985 case Intrinsic::x86_pclmulqdq:
3986 case Intrinsic::x86_pclmulqdq_256:
3987 case Intrinsic::x86_pclmulqdq_512:
3988 handlePclmulIntrinsic(
I);
3991 case Intrinsic::x86_sse41_round_sd:
3992 case Intrinsic::x86_sse41_round_ss:
3993 handleUnarySdSsIntrinsic(
I);
3995 case Intrinsic::x86_sse2_max_sd:
3996 case Intrinsic::x86_sse_max_ss:
3997 case Intrinsic::x86_sse2_min_sd:
3998 case Intrinsic::x86_sse_min_ss:
3999 handleBinarySdSsIntrinsic(
I);
4002 case Intrinsic::x86_avx_vtestc_pd:
4003 case Intrinsic::x86_avx_vtestc_pd_256:
4004 case Intrinsic::x86_avx_vtestc_ps:
4005 case Intrinsic::x86_avx_vtestc_ps_256:
4006 case Intrinsic::x86_avx_vtestnzc_pd:
4007 case Intrinsic::x86_avx_vtestnzc_pd_256:
4008 case Intrinsic::x86_avx_vtestnzc_ps:
4009 case Intrinsic::x86_avx_vtestnzc_ps_256:
4010 case Intrinsic::x86_avx_vtestz_pd:
4011 case Intrinsic::x86_avx_vtestz_pd_256:
4012 case Intrinsic::x86_avx_vtestz_ps:
4013 case Intrinsic::x86_avx_vtestz_ps_256:
4014 case Intrinsic::x86_avx_ptestc_256:
4015 case Intrinsic::x86_avx_ptestnzc_256:
4016 case Intrinsic::x86_avx_ptestz_256:
4017 case Intrinsic::x86_sse41_ptestc:
4018 case Intrinsic::x86_sse41_ptestnzc:
4019 case Intrinsic::x86_sse41_ptestz:
4020 handleVtestIntrinsic(
I);
4023 case Intrinsic::fshl:
4024 case Intrinsic::fshr:
4025 handleFunnelShift(
I);
4028 case Intrinsic::is_constant:
4030 setShadow(&
I, getCleanShadow(&
I));
4031 setOrigin(&
I, getCleanOrigin());
4035 if (!handleUnknownIntrinsic(
I))
4036 visitInstruction(
I);
4041 void visitLibAtomicLoad(
CallBase &CB) {
4043 assert(isa<CallInst>(CB));
4052 Value *NewOrdering =
4056 NextNodeIRBuilder NextIRB(&CB);
4057 Value *SrcShadowPtr, *SrcOriginPtr;
4058 std::tie(SrcShadowPtr, SrcOriginPtr) =
4059 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4061 Value *DstShadowPtr =
4062 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4066 NextIRB.CreateMemCpy(DstShadowPtr,
Align(1), SrcShadowPtr,
Align(1),
Size);
4067 if (MS.TrackOrigins) {
4068 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
4070 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
4071 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
4075 void visitLibAtomicStore(
CallBase &CB) {
4082 Value *NewOrdering =
4086 Value *DstShadowPtr =
4104 visitAsmInstruction(CB);
4106 visitInstruction(CB);
4115 case LibFunc_atomic_load:
4116 if (!isa<CallInst>(CB)) {
4117 llvm::errs() <<
"MSAN -- cannot instrument invoke of libatomic load."
4121 visitLibAtomicLoad(CB);
4123 case LibFunc_atomic_store:
4124 visitLibAtomicStore(CB);
4131 if (
auto *Call = dyn_cast<CallInst>(&CB)) {
4132 assert(!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere");
4140 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4142 Call->removeFnAttrs(
B);
4144 Func->removeFnAttrs(
B);
4150 bool MayCheckCall = MS.EagerChecks;
4154 MayCheckCall &= !
Func->getName().starts_with(
"__sanitizer_unaligned_");
4157 unsigned ArgOffset = 0;
4160 if (!
A->getType()->isSized()) {
4161 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is not sized: " << CB <<
"\n");
4169 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4172 insertShadowCheck(
A, &CB);
4173 Size =
DL.getTypeAllocSize(
A->getType());
4179 Value *ArgShadow = getShadow(
A);
4180 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4182 <<
" Shadow: " << *ArgShadow <<
"\n");
4186 assert(
A->getType()->isPointerTy() &&
4187 "ByVal argument is not a pointer!");
4195 Value *AShadowPtr, *AOriginPtr;
4196 std::tie(AShadowPtr, AOriginPtr) =
4197 getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(), Alignment,
4199 if (!PropagateShadow) {
4206 if (MS.TrackOrigins) {
4207 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4221 Size =
DL.getTypeAllocSize(
A->getType());
4226 Constant *Cst = dyn_cast<Constant>(ArgShadow);
4227 if (MS.TrackOrigins && !(Cst && Cst->
isNullValue())) {
4229 getOriginPtrForArgument(IRB, ArgOffset));
4233 assert(Store !=
nullptr);
4242 if (FT->isVarArg()) {
4243 VAHelper->visitCallBase(CB, IRB);
4250 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
4253 if (MayCheckCall && CB.
hasRetAttr(Attribute::NoUndef)) {
4254 setShadow(&CB, getCleanShadow(&CB));
4255 setOrigin(&CB, getCleanOrigin());
4261 Value *
Base = getShadowPtrForRetval(IRBBefore);
4262 IRBBefore.CreateAlignedStore(getCleanShadow(&CB),
Base,
4265 if (isa<CallInst>(CB)) {
4269 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
4274 setShadow(&CB, getCleanShadow(&CB));
4275 setOrigin(&CB, getCleanOrigin());
4282 "Could not find insertion point for retval shadow load");
4285 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4286 getShadowTy(&CB), getShadowPtrForRetval(IRBAfter),
4288 setShadow(&CB, RetvalShadow);
4289 if (MS.TrackOrigins)
4290 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
4291 getOriginPtrForRetval()));
4295 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
4296 RetVal =
I->getOperand(0);
4298 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
4299 return I->isMustTailCall();
4306 Value *RetVal =
I.getReturnValue();
4312 Value *ShadowPtr = getShadowPtrForRetval(IRB);
4313 bool HasNoUndef =
F.hasRetAttribute(Attribute::NoUndef);
4314 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4317 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (
F.getName() ==
"main");
4319 Value *Shadow = getShadow(RetVal);
4320 bool StoreOrigin =
true;
4322 insertShadowCheck(RetVal, &
I);
4323 Shadow = getCleanShadow(RetVal);
4324 StoreOrigin =
false;
4331 if (MS.TrackOrigins && StoreOrigin)
4332 IRB.
CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
4338 if (!PropagateShadow) {
4339 setShadow(&
I, getCleanShadow(&
I));
4340 setOrigin(&
I, getCleanOrigin());
4344 ShadowPHINodes.push_back(&
I);
4345 setShadow(&
I, IRB.
CreatePHI(getShadowTy(&
I),
I.getNumIncomingValues(),
4347 if (MS.TrackOrigins)
4349 &
I, IRB.
CreatePHI(MS.OriginTy,
I.getNumIncomingValues(),
"_msphi_o"));
4366 IRB.
CreateCall(MS.MsanPoisonStackFn, {&I, Len});
4368 Value *ShadowBase, *OriginBase;
4369 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4376 if (PoisonStack && MS.TrackOrigins) {
4377 Value *Idptr = getLocalVarIdptr(
I);
4379 Value *Descr = getLocalVarDescription(
I);
4380 IRB.
CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4381 {&I, Len, Idptr, Descr});
4383 IRB.
CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
4389 Value *Descr = getLocalVarDescription(
I);
4391 IRB.
CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
4393 IRB.
CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
4400 NextNodeIRBuilder IRB(InsPoint);
4404 if (
I.isArrayAllocation())
4408 if (MS.CompileKernel)
4409 poisonAllocaKmsan(
I, IRB, Len);
4411 poisonAllocaUserspace(
I, IRB, Len);
4415 setShadow(&
I, getCleanShadow(&
I));
4416 setOrigin(&
I, getCleanOrigin());
4428 Value *Sb = getShadow(
B);
4429 Value *Sc = getShadow(
C);
4430 Value *Sd = getShadow(
D);
4435 if (
I.getType()->isAggregateType()) {
4439 Sa1 = getPoisonedShadow(getShadowTy(
I.getType()));
4447 C = CreateAppToShadowCast(IRB,
C);
4448 D = CreateAppToShadowCast(IRB,
D);
4455 if (MS.TrackOrigins) {
4458 if (
B->getType()->isVectorTy()) {
4459 B = convertToBool(
B, IRB);
4460 Sb = convertToBool(Sb, IRB);
4467 getOrigin(
I.getFalseValue()))));
4474 setShadow(&
I, getCleanShadow(&
I));
4475 setOrigin(&
I, getCleanOrigin());
4479 setShadow(&
I, getCleanShadow(&
I));
4480 setOrigin(&
I, getCleanOrigin());
4484 setShadow(&
I, getCleanShadow(&
I));
4485 setOrigin(&
I, getCleanOrigin());
4492 Value *Agg =
I.getAggregateOperand();
4494 Value *AggShadow = getShadow(Agg);
4498 setShadow(&
I, ResShadow);
4499 setOriginForNaryOp(
I);
4505 Value *AggShadow = getShadow(
I.getAggregateOperand());
4506 Value *InsShadow = getShadow(
I.getInsertedValueOperand());
4512 setOriginForNaryOp(
I);
4516 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
4517 errs() <<
"ZZZ call " << CI->getCalledFunction()->getName() <<
"\n";
4519 errs() <<
"ZZZ " <<
I.getOpcodeName() <<
"\n";
4521 errs() <<
"QQQ " <<
I <<
"\n";
4548 insertShadowCheck(Operand, &
I);
4555 auto Size =
DL.getTypeStoreSize(ElemTy);
4557 if (MS.CompileKernel) {
4558 IRB.
CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal});
4564 auto [ShadowPtr,
_] =
4565 getShadowOriginPtrUserspace(Operand, IRB, IRB.
getInt8Ty(),
Align(1));
4576 int NumRetOutputs = 0;
4578 Type *
RetTy = cast<Value>(CB)->getType();
4579 if (!
RetTy->isVoidTy()) {
4581 auto *
ST = dyn_cast<StructType>(
RetTy);
4583 NumRetOutputs =
ST->getNumElements();
4589 switch (
Info.Type) {
4597 return NumOutputs - NumRetOutputs;
4620 int OutputArgs = getNumOutputArgs(IA, CB);
4626 for (
int i = OutputArgs; i < NumOperands; i++) {
4634 for (
int i = 0; i < OutputArgs; i++) {
4640 setShadow(&
I, getCleanShadow(&
I));
4641 setOrigin(&
I, getCleanOrigin());
4646 setShadow(&
I, getCleanShadow(&
I));
4647 setOrigin(&
I, getCleanOrigin());
4655 for (
size_t i = 0, n =
I.getNumOperands(); i < n; i++) {
4656 Value *Operand =
I.getOperand(i);
4658 insertShadowCheck(Operand, &
I);
4660 setShadow(&
I, getCleanShadow(&
I));
4661 setOrigin(&
I, getCleanOrigin());
4665struct VarArgHelperBase :
public VarArgHelper {
4667 MemorySanitizer &MS;
4668 MemorySanitizerVisitor &MSV;
4670 const unsigned VAListTagSize;
4672 VarArgHelperBase(
Function &
F, MemorySanitizer &MS,
4673 MemorySanitizerVisitor &MSV,
unsigned VAListTagSize)
4674 :
F(
F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
4678 return IRB.
CreateAdd(
Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4683 unsigned ArgOffset) {
4692 unsigned ArgOffset,
unsigned ArgSize) {
4696 return getShadowPtrForVAArgument(Ty, IRB, ArgOffset);
4711 unsigned BaseOffset) {
4720 TailSize,
Align(8));
4725 Value *VAListTag =
I.getArgOperand(0);
4727 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
4728 VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
true);
4731 VAListTagSize, Alignment,
false);
4738 unpoisonVAListTagForInst(
I);
4744 unpoisonVAListTagForInst(
I);
4749struct VarArgAMD64Helper :
public VarArgHelperBase {
4752 static const unsigned AMD64GpEndOffset = 48;
4753 static const unsigned AMD64FpEndOffsetSSE = 176;
4755 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4757 unsigned AMD64FpEndOffset;
4760 Value *VAArgOverflowSize =
nullptr;
4762 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4764 VarArgAMD64Helper(
Function &
F, MemorySanitizer &MS,
4765 MemorySanitizerVisitor &MSV)
4766 : VarArgHelperBase(
F, MS, MSV, 24) {
4767 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4768 for (
const auto &Attr :
F.getAttributes().getFnAttrs()) {
4769 if (Attr.isStringAttribute() &&
4770 (Attr.getKindAsString() ==
"target-features")) {
4771 if (Attr.getValueAsString().contains(
"-sse"))
4772 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4778 ArgKind classifyArgument(
Value *arg) {
4781 if (
T->isX86_FP80Ty())
4783 if (
T->isFPOrFPVectorTy() ||
T->isX86_MMXTy())
4784 return AK_FloatingPoint;
4785 if (
T->isIntegerTy() &&
T->getPrimitiveSizeInBits() <= 64)
4786 return AK_GeneralPurpose;
4787 if (
T->isPointerTy())
4788 return AK_GeneralPurpose;
4801 unsigned GpOffset = 0;
4802 unsigned FpOffset = AMD64GpEndOffset;
4803 unsigned OverflowOffset = AMD64FpEndOffset;
4808 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
4815 assert(
A->getType()->isPointerTy());
4817 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
4819 unsigned BaseOffset = OverflowOffset;
4821 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
4822 Value *OriginBase =
nullptr;
4823 if (MS.TrackOrigins)
4824 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4825 OverflowOffset += AlignedSize;
4828 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4832 Value *ShadowPtr, *OriginPtr;
4833 std::tie(ShadowPtr, OriginPtr) =
4838 if (MS.TrackOrigins)
4842 ArgKind AK = classifyArgument(
A);
4843 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4845 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4847 Value *ShadowBase, *OriginBase =
nullptr;
4849 case AK_GeneralPurpose:
4850 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, GpOffset);
4851 if (MS.TrackOrigins)
4852 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset);
4856 case AK_FloatingPoint:
4857 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, FpOffset);
4858 if (MS.TrackOrigins)
4859 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
4866 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
4868 unsigned BaseOffset = OverflowOffset;
4870 getShadowPtrForVAArgument(
A->getType(), IRB, OverflowOffset);
4871 if (MS.TrackOrigins) {
4872 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4874 OverflowOffset += AlignedSize;
4877 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4886 Value *Shadow = MSV.getShadow(
A);
4888 if (MS.TrackOrigins) {
4889 Value *Origin = MSV.getOrigin(
A);
4891 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4897 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
4898 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4901 void finalizeInstrumentation()
override {
4902 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4903 "finalizeInstrumentation called twice");
4904 if (!VAStartInstrumentationList.
empty()) {
4911 ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize);
4918 Intrinsic::umin, CopySize,
4922 if (MS.TrackOrigins) {
4932 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
4933 CallInst *OrigInst = VAStartInstrumentationList[i];
4934 NextNodeIRBuilder IRB(OrigInst);
4937 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
4940 ConstantInt::get(MS.IntptrTy, 16)),
4941 PointerType::get(RegSaveAreaPtrTy, 0));
4942 Value *RegSaveAreaPtr =
4943 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4944 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4946 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4947 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
4949 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4951 if (MS.TrackOrigins)
4952 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
4953 Alignment, AMD64FpEndOffset);
4954 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
4957 ConstantInt::get(MS.IntptrTy, 8)),
4958 PointerType::get(OverflowArgAreaPtrTy, 0));
4959 Value *OverflowArgAreaPtr =
4960 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
4961 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
4962 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
4963 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
4967 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
4969 if (MS.TrackOrigins) {
4972 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
4981struct VarArgMIPS64Helper :
public VarArgHelperBase {
4983 Value *VAArgSize =
nullptr;
4985 VarArgMIPS64Helper(
Function &
F, MemorySanitizer &MS,
4986 MemorySanitizerVisitor &MSV)
4987 : VarArgHelperBase(
F, MS, MSV, 8) {}
4990 unsigned VAArgOffset = 0;
4994 Triple TargetTriple(
F.getParent()->getTargetTriple());
4996 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5001 VAArgOffset += (8 - ArgSize);
5003 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VAArgOffset, ArgSize);
5004 VAArgOffset += ArgSize;
5005 VAArgOffset =
alignTo(VAArgOffset, 8);
5014 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5017 void finalizeInstrumentation()
override {
5018 assert(!VAArgSize && !VAArgTLSCopy &&
5019 "finalizeInstrumentation called twice");
5023 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5025 if (!VAStartInstrumentationList.
empty()) {
5034 Intrinsic::umin, CopySize,
5042 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5043 CallInst *OrigInst = VAStartInstrumentationList[i];
5044 NextNodeIRBuilder IRB(OrigInst);
5046 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5047 Value *RegSaveAreaPtrPtr =
5049 PointerType::get(RegSaveAreaPtrTy, 0));
5050 Value *RegSaveAreaPtr =
5051 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5052 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5054 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5055 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5057 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5064struct VarArgAArch64Helper :
public VarArgHelperBase {
5065 static const unsigned kAArch64GrArgSize = 64;
5066 static const unsigned kAArch64VrArgSize = 128;
5068 static const unsigned AArch64GrBegOffset = 0;
5069 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
5071 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
5072 static const unsigned AArch64VrEndOffset =
5073 AArch64VrBegOffset + kAArch64VrArgSize;
5074 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
5077 Value *VAArgOverflowSize =
nullptr;
5079 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
5081 VarArgAArch64Helper(
Function &
F, MemorySanitizer &MS,
5082 MemorySanitizerVisitor &MSV)
5083 : VarArgHelperBase(
F, MS, MSV, 32) {}
5086 std::pair<ArgKind, uint64_t> classifyArgument(
Type *
T) {
5087 if (
T->isIntOrPtrTy() &&
T->getPrimitiveSizeInBits() <= 64)
5088 return {AK_GeneralPurpose, 1};
5089 if (
T->isFloatingPointTy() &&
T->getPrimitiveSizeInBits() <= 128)
5090 return {AK_FloatingPoint, 1};
5092 if (
T->isArrayTy()) {
5093 auto R = classifyArgument(
T->getArrayElementType());
5094 R.second *=
T->getScalarType()->getArrayNumElements();
5099 auto R = classifyArgument(FV->getScalarType());
5100 R.second *= FV->getNumElements();
5105 return {AK_Memory, 0};
5118 unsigned GrOffset = AArch64GrBegOffset;
5119 unsigned VrOffset = AArch64VrBegOffset;
5120 unsigned OverflowOffset = AArch64VAEndOffset;
5125 auto [AK, RegNum] = classifyArgument(
A->getType());
5126 if (AK == AK_GeneralPurpose &&
5127 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
5129 if (AK == AK_FloatingPoint &&
5130 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
5134 case AK_GeneralPurpose:
5135 Base = getShadowPtrForVAArgument(
A->getType(), IRB, GrOffset);
5136 GrOffset += 8 * RegNum;
5138 case AK_FloatingPoint:
5139 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VrOffset);
5140 VrOffset += 16 * RegNum;
5147 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5149 unsigned BaseOffset = OverflowOffset;
5150 Base = getShadowPtrForVAArgument(
A->getType(), IRB, BaseOffset);
5151 OverflowOffset += AlignedSize;
5154 CleanUnusedTLS(IRB,
Base, BaseOffset);
5166 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
5167 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5174 ConstantInt::get(MS.IntptrTy, offset)),
5175 PointerType::get(*MS.C, 0));
5183 ConstantInt::get(MS.IntptrTy, offset)),
5184 PointerType::get(*MS.C, 0));
5186 return IRB.
CreateSExt(SaveArea32, MS.IntptrTy);
5189 void finalizeInstrumentation()
override {
5190 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5191 "finalizeInstrumentation called twice");
5192 if (!VAStartInstrumentationList.
empty()) {
5199 ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize);
5206 Intrinsic::umin, CopySize,
5212 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
5213 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
5217 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5218 CallInst *OrigInst = VAStartInstrumentationList[i];
5219 NextNodeIRBuilder IRB(OrigInst);
5238 Value *StackSaveAreaPtr =
5239 IRB.
CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy);
5242 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
5243 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
5246 IRB.
CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy);
5249 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
5250 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
5253 IRB.
CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy);
5259 Value *GrRegSaveAreaShadowPtrOff =
5260 IRB.
CreateAdd(GrArgSize, GrOffSaveArea);
5262 Value *GrRegSaveAreaShadowPtr =
5263 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5269 Value *GrCopySize = IRB.
CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
5275 Value *VrRegSaveAreaShadowPtrOff =
5276 IRB.
CreateAdd(VrArgSize, VrOffSaveArea);
5278 Value *VrRegSaveAreaShadowPtr =
5279 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5286 VrRegSaveAreaShadowPtrOff);
5287 Value *VrCopySize = IRB.
CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
5293 Value *StackSaveAreaShadowPtr =
5294 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5299 VAArgTLSCopy, IRB.
getInt32(AArch64VAEndOffset));
5302 Align(16), VAArgOverflowSize);
5308struct VarArgPowerPC64Helper :
public VarArgHelperBase {
5310 Value *VAArgSize =
nullptr;
5312 VarArgPowerPC64Helper(
Function &
F, MemorySanitizer &MS,
5313 MemorySanitizerVisitor &MSV)
5314 : VarArgHelperBase(
F, MS, MSV, 8) {}
5324 Triple TargetTriple(
F.getParent()->getTargetTriple());
5332 unsigned VAArgOffset = VAArgBase;
5336 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
5338 assert(
A->getType()->isPointerTy());
5340 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
5343 ArgAlign =
Align(8);
5344 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5346 Value *
Base = getShadowPtrForVAArgument(
5347 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
5349 Value *AShadowPtr, *AOriginPtr;
5350 std::tie(AShadowPtr, AOriginPtr) =
5351 MSV.getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(),
5361 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5363 if (
A->getType()->isArrayTy()) {
5366 Type *ElementTy =
A->getType()->getArrayElementType();
5368 ArgAlign =
Align(
DL.getTypeAllocSize(ElementTy));
5369 }
else if (
A->getType()->isVectorTy()) {
5371 ArgAlign =
Align(ArgSize);
5374 ArgAlign =
Align(8);
5375 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5376 if (
DL.isBigEndian()) {
5380 VAArgOffset += (8 - ArgSize);
5383 Base = getShadowPtrForVAArgument(
A->getType(), IRB,
5384 VAArgOffset - VAArgBase, ArgSize);
5388 VAArgOffset += ArgSize;
5392 VAArgBase = VAArgOffset;
5396 ConstantInt::get(IRB.
getInt64Ty(), VAArgOffset - VAArgBase);
5399 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5402 void finalizeInstrumentation()
override {
5403 assert(!VAArgSize && !VAArgTLSCopy &&
5404 "finalizeInstrumentation called twice");
5408 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5410 if (!VAStartInstrumentationList.
empty()) {
5420 Intrinsic::umin, CopySize,
5428 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5429 CallInst *OrigInst = VAStartInstrumentationList[i];
5430 NextNodeIRBuilder IRB(OrigInst);
5432 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5433 Value *RegSaveAreaPtrPtr =
5435 PointerType::get(RegSaveAreaPtrTy, 0));
5436 Value *RegSaveAreaPtr =
5437 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5438 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5440 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5441 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5443 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5450struct VarArgSystemZHelper :
public VarArgHelperBase {
5451 static const unsigned SystemZGpOffset = 16;
5452 static const unsigned SystemZGpEndOffset = 56;
5453 static const unsigned SystemZFpOffset = 128;
5454 static const unsigned SystemZFpEndOffset = 160;
5455 static const unsigned SystemZMaxVrArgs = 8;
5456 static const unsigned SystemZRegSaveAreaSize = 160;
5457 static const unsigned SystemZOverflowOffset = 160;
5458 static const unsigned SystemZVAListTagSize = 32;
5459 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5460 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5462 bool IsSoftFloatABI;
5465 Value *VAArgOverflowSize =
nullptr;
5467 enum class ArgKind {
5475 enum class ShadowExtension {
None,
Zero, Sign };
5477 VarArgSystemZHelper(
Function &
F, MemorySanitizer &MS,
5478 MemorySanitizerVisitor &MSV)
5479 : VarArgHelperBase(
F, MS, MSV, SystemZVAListTagSize),
5480 IsSoftFloatABI(
F.getFnAttribute(
"use-soft-float").getValueAsBool()) {}
5482 ArgKind classifyArgument(
Type *
T) {
5489 if (
T->isIntegerTy(128) ||
T->isFP128Ty())
5490 return ArgKind::Indirect;
5491 if (
T->isFloatingPointTy())
5492 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5493 if (
T->isIntegerTy() ||
T->isPointerTy())
5494 return ArgKind::GeneralPurpose;
5495 if (
T->isVectorTy())
5496 return ArgKind::Vector;
5497 return ArgKind::Memory;
5500 ShadowExtension getShadowExtension(
const CallBase &CB,
unsigned ArgNo) {
5510 return ShadowExtension::Zero;
5514 return ShadowExtension::Sign;
5516 return ShadowExtension::None;
5520 unsigned GpOffset = SystemZGpOffset;
5521 unsigned FpOffset = SystemZFpOffset;
5522 unsigned VrIndex = 0;
5523 unsigned OverflowOffset = SystemZOverflowOffset;
5530 ArgKind AK = classifyArgument(
T);
5531 if (AK == ArgKind::Indirect) {
5532 T = PointerType::get(
T, 0);
5533 AK = ArgKind::GeneralPurpose;
5535 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5536 AK = ArgKind::Memory;
5537 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5538 AK = ArgKind::Memory;
5539 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5540 AK = ArgKind::Memory;
5541 Value *ShadowBase =
nullptr;
5542 Value *OriginBase =
nullptr;
5543 ShadowExtension SE = ShadowExtension::None;
5545 case ArgKind::GeneralPurpose: {
5550 SE = getShadowExtension(CB, ArgNo);
5552 if (SE == ShadowExtension::None) {
5554 assert(ArgAllocSize <= ArgSize);
5555 GapSize = ArgSize - ArgAllocSize;
5557 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5558 if (MS.TrackOrigins)
5559 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5561 GpOffset += ArgSize;
5567 case ArgKind::FloatingPoint: {
5576 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5577 if (MS.TrackOrigins)
5578 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5580 FpOffset += ArgSize;
5586 case ArgKind::Vector: {
5593 case ArgKind::Memory: {
5601 SE = getShadowExtension(CB, ArgNo);
5603 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5605 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5606 if (MS.TrackOrigins)
5608 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5609 OverflowOffset += ArgSize;
5616 case ArgKind::Indirect:
5619 if (ShadowBase ==
nullptr)
5621 Value *Shadow = MSV.getShadow(
A);
5622 if (SE != ShadowExtension::None)
5623 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.
getInt64Ty(),
5624 SE == ShadowExtension::Sign);
5626 ShadowBase, PointerType::get(Shadow->
getType(), 0),
"_msarg_va_s");
5628 if (MS.TrackOrigins) {
5629 Value *Origin = MSV.getOrigin(
A);
5631 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5635 Constant *OverflowSize = ConstantInt::get(
5636 IRB.
getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5637 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5641 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5645 ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5646 PointerType::get(RegSaveAreaPtrTy, 0));
5647 Value *RegSaveAreaPtr = IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5648 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5650 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5651 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(), Alignment,
5656 unsigned RegSaveAreaSize =
5657 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
5658 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5660 if (MS.TrackOrigins)
5661 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5662 Alignment, RegSaveAreaSize);
5668 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5672 ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5673 PointerType::get(OverflowArgAreaPtrTy, 0));
5674 Value *OverflowArgAreaPtr =
5675 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5676 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5678 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5679 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5682 SystemZOverflowOffset);
5683 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5685 if (MS.TrackOrigins) {
5687 SystemZOverflowOffset);
5688 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5693 void finalizeInstrumentation()
override {
5694 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5695 "finalizeInstrumentation called twice");
5696 if (!VAStartInstrumentationList.
empty()) {
5703 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5711 Intrinsic::umin, CopySize,
5715 if (MS.TrackOrigins) {
5725 for (
size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.
size();
5726 VaStartNo < VaStartNum; VaStartNo++) {
5727 CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5728 NextNodeIRBuilder IRB(OrigInst);
5730 copyRegSaveArea(IRB, VAListTag);
5731 copyOverflowArea(IRB, VAListTag);
5738using VarArgLoongArch64Helper = VarArgMIPS64Helper;
5741struct VarArgNoOpHelper :
public VarArgHelper {
5742 VarArgNoOpHelper(
Function &
F, MemorySanitizer &MS,
5743 MemorySanitizerVisitor &MSV) {}
5751 void finalizeInstrumentation()
override {}
5757 MemorySanitizerVisitor &Visitor) {
5760 Triple TargetTriple(Func.getParent()->getTargetTriple());
5762 return new VarArgAMD64Helper(Func, Msan, Visitor);
5764 return new VarArgMIPS64Helper(Func, Msan, Visitor);
5766 return new VarArgAArch64Helper(Func, Msan, Visitor);
5769 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5771 return new VarArgSystemZHelper(Func, Msan, Visitor);
5773 return new VarArgLoongArch64Helper(Func, Msan, Visitor);
5775 return new VarArgNoOpHelper(Func, Msan, Visitor);
5782 if (
F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
5785 MemorySanitizerVisitor Visitor(
F, *
this, TLI);
5789 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
5792 return Visitor.runOnFunction();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
static bool isStore(int Opcode)
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
static const size_t kNumberOfAccessSizes
VarLocInsertPt getNextNode(const DbgRecord *DPV)
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AtomicOrdering addReleaseOrdering(AtomicOrdering AO)
static AtomicOrdering addAcquireOrdering(AtomicOrdering AO)
static bool isAMustTailRetVal(Value *RetVal)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool runOnFunction(Function &F, bool PostInlining)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
static size_t TypeSizeToSizeIndex(uint32_t TypeSize)
static const MemoryMapParams Linux_LoongArch64_MemoryMapParams
static const PlatformMemoryMapParams Linux_S390_MemoryMapParams
static const Align kMinOriginAlignment
static const MemoryMapParams Linux_X86_64_MemoryMapParams
static cl::opt< uint64_t > ClShadowBase("msan-shadow-base", cl::desc("Define custom MSan ShadowBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClDumpStrictInstructions("msan-dump-strict-instructions", cl::desc("print out instructions with default strict semantics"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClHandleICmpExact("msan-handle-icmp-exact", cl::desc("exact handling of relational integer ICmp"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams Linux_X86_MemoryMapParams
static cl::opt< uint64_t > ClOriginBase("msan-origin-base", cl::desc("Define custom MSan OriginBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClCheckConstantShadow("msan-check-constant-shadow", cl::desc("Insert checks for constant shadow values"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams
static const MemoryMapParams NetBSD_X86_64_MemoryMapParams
static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams
static const unsigned kOriginSize
static cl::opt< bool > ClWithComdat("msan-with-comdat", cl::desc("Place MSan constructors in comdat sections"), cl::Hidden, cl::init(false))
static cl::opt< int > ClTrackOrigins("msan-track-origins", cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden, cl::init(0))
Track origins of uninitialized values.
static cl::opt< int > ClInstrumentationWithCallThreshold("msan-instrumentation-with-call-threshold", cl::desc("If the function being instrumented requires more than " "this number of checks and origin stores, use callbacks instead of " "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500))
static cl::opt< int > ClPoisonStackPattern("msan-poison-stack-pattern", cl::desc("poison uninitialized stack variables with the given pattern"), cl::Hidden, cl::init(0xff))
static const Align kShadowTLSAlignment
static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams
static Constant * getOrInsertGlobal(Module &M, StringRef Name, Type *Ty)
static const MemoryMapParams Linux_S390X_MemoryMapParams
static cl::opt< bool > ClPoisonUndef("msan-poison-undef", cl::desc("poison undef temps"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClPoisonStack("msan-poison-stack", cl::desc("poison uninitialized stack variables"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_I386_MemoryMapParams
const char kMsanInitName[]
static cl::opt< bool > ClPrintStackNames("msan-print-stack-names", cl::desc("Print name of local stack variable"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_AArch64_MemoryMapParams
static cl::opt< uint64_t > ClAndMask("msan-and-mask", cl::desc("Define custom MSan AndMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleLifetimeIntrinsics("msan-handle-lifetime-intrinsics", cl::desc("when possible, poison scoped variables at the beginning of the scope " "(slower, but more precise)"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClKeepGoing("msan-keep-going", cl::desc("keep going after reporting a UMR"), cl::Hidden, cl::init(false))
static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams
static GlobalVariable * createPrivateConstGlobalForString(Module &M, StringRef Str)
Create a non-const global initialized with the given string.
static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams
static const size_t kNumberOfAccessSizes
static cl::opt< bool > ClEagerChecks("msan-eager-checks", cl::desc("check arguments and return values at function call boundaries"), cl::Hidden, cl::init(false))
static cl::opt< int > ClDisambiguateWarning("msan-disambiguate-warning-threshold", cl::desc("Define threshold for number of checks per " "debug location to force origin update."), cl::Hidden, cl::init(3))
static VarArgHelper * CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, MemorySanitizerVisitor &Visitor)
static const MemoryMapParams Linux_MIPS64_MemoryMapParams
static const MemoryMapParams Linux_PowerPC64_MemoryMapParams
static cl::opt< uint64_t > ClXorMask("msan-xor-mask", cl::desc("Define custom MSan XorMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleAsmConservative("msan-handle-asm-conservative", cl::desc("conservative handling of inline assembly"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams
static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams
static const unsigned kParamTLSSize
static cl::opt< bool > ClHandleICmp("msan-handle-icmp", cl::desc("propagate shadow through ICmpEQ and ICmpNE"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClEnableKmsan("msan-kernel", cl::desc("Enable KernelMemorySanitizer instrumentation"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClPoisonStackWithCall("msan-poison-stack-with-call", cl::desc("poison uninitialized stack variables with a call"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams
static const unsigned kRetvalTLSSize
static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams
const char kMsanModuleCtorName[]
static const MemoryMapParams FreeBSD_I386_MemoryMapParams
static cl::opt< bool > ClCheckAccessAddress("msan-check-access-address", cl::desc("report accesses through a pointer which has poisoned shadow"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClDisableChecks("msan-disable-checks", cl::desc("Apply no_sanitize to the whole file"), cl::Hidden, cl::init(false))
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
InstListType::iterator iterator
Instruction iterators...
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Type * getParamByValType(unsigned ArgNo) const
Extract the byval type for a call or parameter.
Value * getCalledOperand() const
Type * getParamElementType(unsigned ArgNo) const
Extract the elementtype type for a parameter.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_SGT
signed greater than
@ ICMP_SGE
signed greater or equal
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
bool isZeroValue() const
Return true if the value is negative zero or null value.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
static bool shouldExecute(unsigned CounterName)
This instruction compares its operands according to the predicate given to the constructor.
Class to represent fixed width SIMD vectors.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void setComdat(Comdat *C)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ ExternalLinkage
Externally visible function.
Analysis pass providing a never-invalidated alias analysis result.
This instruction compares its operands according to the predicate given to the constructor.
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTypeSize(Type *DstType, TypeSize Size)
Create an expression which evaluates to the number of units in Size at runtime.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::vector< ConstraintInfo > ConstraintInfoVector
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
This class represents a cast from an integer to a pointer.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
This class wraps the llvm.memcpy intrinsic.
This class wraps the llvm.memmove intrinsic.
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
This class represents a cast from a pointer to an integer.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This instruction constructs a fixed permutation of two input vectors.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
AttributeList getAttrList(LLVMContext *C, ArrayRef< unsigned > ArgNos, bool Signed, bool Ret=false, AttributeList AL=AttributeList()) const
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Triple - Helper class for working with autoconf configuration names.
bool isMIPS64() const
Tests whether the target is MIPS 64-bit (little and big endian).
ArchType getArch() const
Get the parsed architecture type of this triple.
bool isLoongArch64() const
Tests whether the target is 64-bit LoongArch.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
static Type * getX86_MMXTy(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
'undef' values are things that do not have specified contents.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This represents the llvm.va_copy intrinsic.
This represents the llvm.va_start intrinsic.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
This class represents zero extension of integer types.
int getNumOccurrences() const
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
This class provides various memory handling functions that manipulate MemoryBlock instances.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
initializer< Ty > init(const Ty &Val)
Function * Kernel
Summary of a kernel (=entry point for target offloading).
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
bool isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to be non-zero when defined.
void stable_sort(R &&Range)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
AllocaInst * findAllocaForValue(Value *V, bool OffsetZero=false)
Returns unique alloca where the value comes from, or nullptr.
std::pair< Function *, FunctionCallee > getOrCreateSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, function_ref< void(Function *, FunctionCallee)> FunctionsCreatedCallback, StringRef VersionCheckName=StringRef(), bool Weak=false)
Creates sanitizer constructor function lazily.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Or
Bitwise or logical OR of integers.
std::pair< Instruction *, Value * > SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore)
Insert a for (int i = 0; i < End; i++) loop structure (with the exception that End is assumed > 0,...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
iterator_range< df_iterator< T > > depth_first(const T &G)
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, const TargetLibraryInfo *TLI)
Given a CallInst, check if it calls a string function known to CodeGen, and mark it with NoBuiltin if...
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A CRTP mix-in to automatically provide informational APIs needed for passes.