33 #define DEBUG_TYPE "lowerbitsets"
35 STATISTIC(ByteArraySizeBits,
"Byte array size in bits");
36 STATISTIC(ByteArraySizeBytes,
"Byte array size in bytes");
37 STATISTIC(NumByteArraysCreated,
"Number of byte arrays created");
38 STATISTIC(NumBitSetCallsLowered,
"Number of bitset calls lowered");
39 STATISTIC(NumBitSetDisjointSets,
"Number of disjoint sets of bitsets");
42 "lowerbitsets-avoid-reuse",
43 cl::desc(
"Try to avoid reuse of byte array addresses using aliases"),
57 return Bits.count(BitOffset);
63 uint64_t COffset)
const {
64 if (
auto GV = dyn_cast<GlobalVariable>(V)) {
65 auto I = GlobalLayout.
find(GV);
66 if (
I == GlobalLayout.
end())
71 if (
auto GEP = dyn_cast<GEPOperator>(V)) {
73 bool Result =
GEP->accumulateConstantOffset(DL, APOffset);
76 COffset += APOffset.getZExtValue();
81 if (
auto Op = dyn_cast<Operator>(V)) {
82 if (Op->getOpcode() == Instruction::BitCast)
83 return containsValue(DL, GlobalLayout, Op->getOperand(0), COffset);
86 return containsValue(DL, GlobalLayout, Op->getOperand(1), COffset) &&
103 for (uint64_t &Offset :
Offsets) {
118 for (uint64_t Offset : Offsets) {
120 BSI.
Bits.insert(Offset);
129 std::vector<uint64_t> &Fragment =
Fragments.back();
130 uint64_t FragmentIndex =
Fragments.size() - 1;
132 for (
auto ObjIndex : F) {
134 if (OldFragmentIndex == 0) {
137 Fragment.push_back(ObjIndex);
144 std::vector<uint64_t> &OldFragment =
Fragments[OldFragmentIndex];
145 Fragment.insert(Fragment.end(), OldFragment.begin(), OldFragment.end());
151 for (uint64_t ObjIndex : Fragment)
156 uint64_t BitSize, uint64_t &AllocByteOffset,
157 uint8_t &AllocMask) {
167 unsigned ReqSize = AllocByteOffset + BitSize;
169 if (
Bytes.size() < ReqSize)
170 Bytes.resize(ReqSize);
173 AllocMask = 1 <<
Bit;
174 for (uint64_t B : Bits)
175 Bytes[AllocByteOffset + B] |= AllocMask;
180 struct ByteArrayInfo {
181 std::set<uint64_t>
Bits;
195 bool LinkerSubsectionsViaSymbols;
209 std::vector<ByteArrayInfo> ByteArrayInfos;
214 ByteArrayInfo *createByteArray(
BitSetInfo &BSI);
215 void allocateByteArrays();
222 void buildBitSetsFromGlobals(
const std::vector<MDString *> &BitSets,
223 const std::vector<GlobalVariable *> &Globals);
225 bool eraseBitSetMetadata();
227 bool doInitialization(
Module &M)
override;
228 bool runOnModule(
Module &M)
override;
234 "Lower bitset metadata",
false,
false)
237 char LowerBitSets::
ID = 0;
241 bool LowerBitSets::doInitialization(
Module &Mod) {
245 Triple TargetTriple(M->getTargetTriple());
246 LinkerSubsectionsViaSymbols = TargetTriple.
isMacOSX();
255 BitSetNM = M->getNamedMetadata(
"llvm.bitsets");
257 BitSetTestCallSites.clear();
272 if (Op->getOperand(0) != BitSet || !Op->getOperand(1))
275 cast<ConstantAsMetadata>(Op->getOperand(1))->getValue());
279 cast<ConstantInt>(cast<ConstantAsMetadata>(Op->getOperand(2))
280 ->getValue())->getZExtValue();
282 Offset += GlobalLayout.
find(OpGlobal)->second;
295 auto BitsType = cast<IntegerType>(Bits->
getType());
296 unsigned BitWidth = BitsType->getBitWidth();
306 ByteArrayInfo *LowerBitSets::createByteArray(
BitSetInfo &BSI) {
315 ByteArrayInfos.emplace_back();
316 ByteArrayInfo *BAI = &ByteArrayInfos.back();
318 BAI->Bits = BSI.
Bits;
320 BAI->ByteArray = ByteArrayGlobal;
325 void LowerBitSets::allocateByteArrays() {
326 std::stable_sort(ByteArrayInfos.begin(), ByteArrayInfos.end(),
327 [](
const ByteArrayInfo &BAI1,
const ByteArrayInfo &BAI2) {
328 return BAI1.BitSize > BAI2.BitSize;
331 std::vector<uint64_t> ByteArrayOffsets(ByteArrayInfos.size());
334 for (
unsigned I = 0;
I != ByteArrayInfos.size(); ++
I) {
335 ByteArrayInfo *BAI = &ByteArrayInfos[
I];
338 BAB.allocate(BAI->Bits, BAI->BitSize, ByteArrayOffsets[
I], Mask);
341 cast<GlobalVariable>(BAI->Mask->getOperand(0))->eraseFromParent();
349 for (
unsigned I = 0; I != ByteArrayInfos.size(); ++
I) {
350 ByteArrayInfo *BAI = &ByteArrayInfos[
I];
355 ByteArrayConst->
getType(), ByteArray, Idxs);
360 if (LinkerSubsectionsViaSymbols) {
361 BAI->ByteArray->replaceAllUsesWith(GEP);
366 BAI->ByteArray->replaceAllUsesWith(Alias);
368 BAI->ByteArray->eraseFromParent();
371 ByteArraySizeBits = BAB.BitAllocs[0] + BAB.BitAllocs[1] + BAB.BitAllocs[2] +
372 BAB.BitAllocs[3] + BAB.BitAllocs[4] + BAB.BitAllocs[5] +
373 BAB.BitAllocs[6] + BAB.BitAllocs[7];
374 ByteArraySizeBytes = BAB.Bytes.size();
380 ByteArrayInfo *&BAI,
Value *BitOffset) {
392 Bits |= uint64_t(1) <<
Bit;
397 ++NumByteArraysCreated;
398 BAI = createByteArray(BSI);
401 Constant *ByteArray = BAI->ByteArray;
402 Type *Ty = BAI->ByteArray->getValueType();
403 if (!LinkerSubsectionsViaSymbols &&
AvoidReuse) {
422 Value *LowerBitSets::lowerBitSetCall(
449 BitOffset = PtrOffset;
464 BitOffset = B.
CreateOr(OffsetSHR, OffsetSHL);
472 return OffsetInRange;
479 Value *
Bit = createBitSetTest(ThenB, BSI, BAI, BitOffset);
493 void LowerBitSets::buildBitSetsFromGlobals(
494 const std::vector<MDString *> &BitSets,
495 const std::vector<GlobalVariable *> &Globals) {
497 std::vector<Constant *> GlobalInits;
500 GlobalInits.push_back(
G->getInitializer());
505 uint64_t Padding =
NextPowerOf2(InitSize - 1) - InitSize;
512 GlobalInits.push_back(
515 if (!GlobalInits.empty())
516 GlobalInits.pop_back();
518 auto CombinedGlobal =
527 for (
unsigned I = 0; I != Globals.size(); ++
I)
529 GlobalLayout[Globals[I]] = CombinedGlobalLayout->getElementOffset(I * 2);
534 BitSetInfo BSI = buildBitSet(BS, GlobalLayout);
536 ByteArrayInfo *BAI = 0;
539 for (
CallInst *CI : BitSetTestCallSites[BS]) {
540 ++NumBitSetCallsLowered;
541 Value *Lowered = lowerBitSetCall(CI, BSI, BAI, CombinedGlobal, GlobalLayout);
550 for (
unsigned I = 0; I != Globals.size(); ++
I) {
555 NewInit->
getType(), CombinedGlobal, CombinedGlobalIdxs);
556 if (LinkerSubsectionsViaSymbols) {
561 "", CombinedGlobalElemPtr, M);
563 Globals[
I]->replaceAllUsesWith(GAlias);
565 Globals[
I]->eraseFromParent();
570 bool LowerBitSets::buildBitSets() {
581 GlobalClassesTy GlobalClasses;
583 for (
const Use &U : BitSetTestFunc->
uses()) {
584 auto CI = cast<CallInst>(U.getUser());
587 if (!BitSetMDVal || !isa<MDString>(BitSetMDVal->getMetadata()))
589 "Second argument of llvm.bitset.test must be metadata string");
590 auto BitSet = cast<MDString>(BitSetMDVal->getMetadata());
596 std::pair<DenseMap<MDString *, std::vector<CallInst *>>::iterator,
598 BitSetTestCallSites.insert(
599 std::make_pair(BitSet, std::vector<CallInst *>()));
600 Ins.first->second.push_back(CI);
605 GlobalClassesTy::iterator GCI = GlobalClasses.insert(BitSet);
606 GlobalClassesTy::member_iterator CurSet = GlobalClasses.findLeader(GCI);
614 if (Op->getNumOperands() != 3)
616 "All operands of llvm.bitsets metadata must have 3 elements");
618 if (Op->getOperand(0) != BitSet || !Op->getOperand(1))
634 "Bit set element offset must be an integer constant");
636 CurSet = GlobalClasses.unionSets(
637 CurSet, GlobalClasses.findLeader(GlobalClasses.insert(OpGlobal)));
641 if (GlobalClasses.empty())
645 for (GlobalClassesTy::iterator I = GlobalClasses.begin(),
646 E = GlobalClasses.end();
648 if (!I->isLeader())
continue;
650 ++NumBitSetDisjointSets;
653 std::vector<MDString *> BitSets;
654 std::vector<GlobalVariable *> Globals;
657 for (GlobalClassesTy::member_iterator
MI = GlobalClasses.member_begin(I);
658 MI != GlobalClasses.member_end(); ++
MI) {
670 std::vector<std::set<uint64_t>> BitSetMembers(BitSets.size());
674 if (!Op->getOperand(1))
676 auto I = BitSetIndices.
find(cast<MDString>(Op->getOperand(0)));
677 if (I == BitSetIndices.
end())
681 cast<ConstantAsMetadata>(Op->getOperand(1))->getValue());
684 BitSetMembers[I->second].insert(GlobalIndices[OpGlobal]);
691 BitSetMembers.begin(), BitSetMembers.end(),
692 [](
const std::set<uint64_t> &O1,
const std::set<uint64_t> &O2) {
693 return O1.size() < O2.size();
700 for (
auto &&MemSet : BitSetMembers)
704 std::vector<GlobalVariable *> OrderedGlobals(Globals.size());
705 auto OGI = OrderedGlobals.begin();
706 for (
auto &&
F : GLB.Fragments)
707 for (
auto &&Offset :
F)
708 *OGI++ = Globals[Offset];
712 return S1->
getString() < S2->getString();
716 buildBitSetsFromGlobals(BitSets, OrderedGlobals);
719 allocateByteArrays();
724 bool LowerBitSets::eraseBitSetMetadata() {
728 M->eraseNamedMetadata(BitSetNM);
732 bool LowerBitSets::runOnModule(
Module &M) {
733 bool Changed = buildBitSets();
734 Changed |= eraseBitSetMetadata();
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
A parsed version of the target data layout string in and methods for querying it. ...
iterator_range< use_iterator > uses()
static IntegerType * getInt1Ty(LLVMContext &C)
LoadInst * CreateLoad(Value *Ptr, const char *Name)
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
This class is used to build a byte array containing overlapping bit sets.
STATISTIC(NumFunctions,"Total number of functions")
A Module instance is used to store all the information related to an LLVM module. ...
static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)
getAnon - Return an anonymous struct that has the specified elements.
static ConstantAggregateZero * get(Type *Ty)
CallInst - This class represents a function call, abstracting a target machine's calling convention...
TerminatorInst * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DominatorTree *DT=nullptr)
SplitBlockAndInsertIfThen - Split the containing block at the specified instruction - everything befo...
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, bool InBounds=false, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
std::vector< uint8_t > Bytes
The byte array built so far.
Like Internal, but omit from symbol table.
bool containsValue(const DataLayout &DL, const DenseMap< GlobalVariable *, uint64_t > &GlobalLayout, Value *V, uint64_t COffset=0) const
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
static IntegerType * getInt64Ty(LLVMContext &C)
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
void allocate(const std::set< uint64_t > &Bits, uint64_t BitSize, uint64_t &AllocByteOffset, uint8_t &AllocMask)
Allocate BitSize bits in the byte array where Bits contains the bits to set.
SmallVector< uint64_t, 16 > Offsets
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
static Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
bool isMacOSX() const
isMacOSX - Is this a Mac OS X triple.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with array type with an element count and element type matchin...
A Use represents the edge between a Value definition and its users.
The returned value is undefined.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
INITIALIZE_PASS_BEGIN(LowerBitSets,"lowerbitsets","Lower bitset metadata", false, false) INITIALIZE_PASS_END(LowerBitSets
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
std::vector< std::vector< uint64_t > > Fragments
The computed layout.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
void addFragment(const std::set< uint64_t > &F)
Add F to the layout while trying to keep its indices contiguous.
void takeName(Value *V)
Transfer the name from V to this value.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
std::vector< uint64_t > FragmentMap
Mapping from object index to fragment index.
initializer< Ty > init(const Ty &Val)
static Constant * getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList)
Create an "inbounds" getelementptr.
Subclasses of this class are all able to terminate a basic block.
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
uint64_t BitAllocs[BitsPerByte]
The number of bytes allocated so far for each of the bits.
static GlobalAlias * create(PointerType *Ty, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
This is an important base class in LLVM.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ModulePass * createLowerBitSetsPass()
This pass lowers bitset metadata and the llvm.bitset.test intrinsic to bitsets.
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
Class to represent integer types.
bool isSingleOffset() const
uint64_t NextPowerOf2(uint64_t A)
NextPowerOf2 - Returns the next power of two (in 64-bits) that is strictly greater than A...
std::set< uint64_t > Bits
Triple - Helper class for working with autoconf configuration names.
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
StringRef getString() const
static PointerType * getUnqual(Type *ElementType)
PointerType::getUnqual - This constructs a pointer to an object of the specified type in the generic ...
This is the shared class of boolean and integer constants.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static ConstantInt * getTrue(LLVMContext &Context)
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
Class for arbitrary precision integers.
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
uint64_t RoundUpToAlignment(uint64_t Value, uint64_t Align)
Returns the next integer (mod 2**64) that is greater than or equal to Value and is a multiple of Alig...
This class implements a layout algorithm for globals referenced by bit sets that tries to keep member...
void initializeLowerBitSetsPass(PassRegistry &)
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
static cl::opt< bool > AvoidReuse("lowerbitsets-avoid-reuse", cl::desc("Try to avoid reuse of byte array addresses using aliases"), cl::Hidden, cl::init(true))
std::string getName(ID id, ArrayRef< Type * > Tys=None)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
static IntegerType * getInt32Ty(LLVMContext &C)
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
iterator find(const KeyT &Val)
static ArrayType * get(Type *ElementType, uint64_t NumElements)
ArrayType::get - This static method is the primary way to construct an ArrayType. ...
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
void addOffset(uint64_t Offset)
op_range operands() const
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
static Value * createMaskedBitTest(IRBuilder<> &B, Value *Bits, Value *BitOffset)
Build a test that bit BitOffset mod sizeof(Bits)*8 is set in Bits.
static IntegerType * getInt8Ty(LLVMContext &C)
bool containsGlobalOffset(uint64_t Offset) const
const BasicBlock * getParent() const
LLVMContext & getContext() const
Get the global data context.