LLVM  3.7.0
LowerBitSets.cpp
Go to the documentation of this file.
1 //===-- LowerBitSets.cpp - Bitset lowering pass ---------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers bitset metadata and calls to the llvm.bitset.test intrinsic.
11 // See http://llvm.org/docs/LangRef.html#bitsets for more information.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/Transforms/IPO.h"
18 #include "llvm/ADT/Statistic.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/IR/Constant.h"
21 #include "llvm/IR/Constants.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IRBuilder.h"
24 #include "llvm/IR/Instructions.h"
25 #include "llvm/IR/Intrinsics.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/IR/Operator.h"
28 #include "llvm/Pass.h"
30 
31 using namespace llvm;
32 
33 #define DEBUG_TYPE "lowerbitsets"
34 
35 STATISTIC(ByteArraySizeBits, "Byte array size in bits");
36 STATISTIC(ByteArraySizeBytes, "Byte array size in bytes");
37 STATISTIC(NumByteArraysCreated, "Number of byte arrays created");
38 STATISTIC(NumBitSetCallsLowered, "Number of bitset calls lowered");
39 STATISTIC(NumBitSetDisjointSets, "Number of disjoint sets of bitsets");
40 
42  "lowerbitsets-avoid-reuse",
43  cl::desc("Try to avoid reuse of byte array addresses using aliases"),
44  cl::Hidden, cl::init(true));
45 
46 bool BitSetInfo::containsGlobalOffset(uint64_t Offset) const {
47  if (Offset < ByteOffset)
48  return false;
49 
50  if ((Offset - ByteOffset) % (uint64_t(1) << AlignLog2) != 0)
51  return false;
52 
53  uint64_t BitOffset = (Offset - ByteOffset) >> AlignLog2;
54  if (BitOffset >= BitSize)
55  return false;
56 
57  return Bits.count(BitOffset);
58 }
59 
61  const DataLayout &DL,
62  const DenseMap<GlobalVariable *, uint64_t> &GlobalLayout, Value *V,
63  uint64_t COffset) const {
64  if (auto GV = dyn_cast<GlobalVariable>(V)) {
65  auto I = GlobalLayout.find(GV);
66  if (I == GlobalLayout.end())
67  return false;
68  return containsGlobalOffset(I->second + COffset);
69  }
70 
71  if (auto GEP = dyn_cast<GEPOperator>(V)) {
72  APInt APOffset(DL.getPointerSizeInBits(0), 0);
73  bool Result = GEP->accumulateConstantOffset(DL, APOffset);
74  if (!Result)
75  return false;
76  COffset += APOffset.getZExtValue();
77  return containsValue(DL, GlobalLayout, GEP->getPointerOperand(),
78  COffset);
79  }
80 
81  if (auto Op = dyn_cast<Operator>(V)) {
82  if (Op->getOpcode() == Instruction::BitCast)
83  return containsValue(DL, GlobalLayout, Op->getOperand(0), COffset);
84 
85  if (Op->getOpcode() == Instruction::Select)
86  return containsValue(DL, GlobalLayout, Op->getOperand(1), COffset) &&
87  containsValue(DL, GlobalLayout, Op->getOperand(2), COffset);
88  }
89 
90  return false;
91 }
92 
94  if (Min > Max)
95  Min = 0;
96 
97  // Normalize each offset against the minimum observed offset, and compute
98  // the bitwise OR of each of the offsets. The number of trailing zeros
99  // in the mask gives us the log2 of the alignment of all offsets, which
100  // allows us to compress the bitset by only storing one bit per aligned
101  // address.
102  uint64_t Mask = 0;
103  for (uint64_t &Offset : Offsets) {
104  Offset -= Min;
105  Mask |= Offset;
106  }
107 
108  BitSetInfo BSI;
109  BSI.ByteOffset = Min;
110 
111  BSI.AlignLog2 = 0;
112  if (Mask != 0)
114 
115  // Build the compressed bitset while normalizing the offsets against the
116  // computed alignment.
117  BSI.BitSize = ((Max - Min) >> BSI.AlignLog2) + 1;
118  for (uint64_t Offset : Offsets) {
119  Offset >>= BSI.AlignLog2;
120  BSI.Bits.insert(Offset);
121  }
122 
123  return BSI;
124 }
125 
126 void GlobalLayoutBuilder::addFragment(const std::set<uint64_t> &F) {
127  // Create a new fragment to hold the layout for F.
128  Fragments.emplace_back();
129  std::vector<uint64_t> &Fragment = Fragments.back();
130  uint64_t FragmentIndex = Fragments.size() - 1;
131 
132  for (auto ObjIndex : F) {
133  uint64_t OldFragmentIndex = FragmentMap[ObjIndex];
134  if (OldFragmentIndex == 0) {
135  // We haven't seen this object index before, so just add it to the current
136  // fragment.
137  Fragment.push_back(ObjIndex);
138  } else {
139  // This index belongs to an existing fragment. Copy the elements of the
140  // old fragment into this one and clear the old fragment. We don't update
141  // the fragment map just yet, this ensures that any further references to
142  // indices from the old fragment in this fragment do not insert any more
143  // indices.
144  std::vector<uint64_t> &OldFragment = Fragments[OldFragmentIndex];
145  Fragment.insert(Fragment.end(), OldFragment.begin(), OldFragment.end());
146  OldFragment.clear();
147  }
148  }
149 
150  // Update the fragment map to point our object indices to this fragment.
151  for (uint64_t ObjIndex : Fragment)
152  FragmentMap[ObjIndex] = FragmentIndex;
153 }
154 
155 void ByteArrayBuilder::allocate(const std::set<uint64_t> &Bits,
156  uint64_t BitSize, uint64_t &AllocByteOffset,
157  uint8_t &AllocMask) {
158  // Find the smallest current allocation.
159  unsigned Bit = 0;
160  for (unsigned I = 1; I != BitsPerByte; ++I)
161  if (BitAllocs[I] < BitAllocs[Bit])
162  Bit = I;
163 
164  AllocByteOffset = BitAllocs[Bit];
165 
166  // Add our size to it.
167  unsigned ReqSize = AllocByteOffset + BitSize;
168  BitAllocs[Bit] = ReqSize;
169  if (Bytes.size() < ReqSize)
170  Bytes.resize(ReqSize);
171 
172  // Set our bits.
173  AllocMask = 1 << Bit;
174  for (uint64_t B : Bits)
175  Bytes[AllocByteOffset + B] |= AllocMask;
176 }
177 
178 namespace {
179 
180 struct ByteArrayInfo {
181  std::set<uint64_t> Bits;
182  uint64_t BitSize;
183  GlobalVariable *ByteArray;
184  Constant *Mask;
185 };
186 
187 struct LowerBitSets : public ModulePass {
188  static char ID;
189  LowerBitSets() : ModulePass(ID) {
191  }
192 
193  Module *M;
194 
195  bool LinkerSubsectionsViaSymbols;
196  IntegerType *Int1Ty;
197  IntegerType *Int8Ty;
198  IntegerType *Int32Ty;
199  Type *Int32PtrTy;
200  IntegerType *Int64Ty;
201  Type *IntPtrTy;
202 
203  // The llvm.bitsets named metadata.
204  NamedMDNode *BitSetNM;
205 
206  // Mapping from bitset mdstrings to the call sites that test them.
208 
209  std::vector<ByteArrayInfo> ByteArrayInfos;
210 
211  BitSetInfo
212  buildBitSet(MDString *BitSet,
213  const DenseMap<GlobalVariable *, uint64_t> &GlobalLayout);
214  ByteArrayInfo *createByteArray(BitSetInfo &BSI);
215  void allocateByteArrays();
216  Value *createBitSetTest(IRBuilder<> &B, BitSetInfo &BSI, ByteArrayInfo *&BAI,
217  Value *BitOffset);
218  Value *
219  lowerBitSetCall(CallInst *CI, BitSetInfo &BSI, ByteArrayInfo *&BAI,
220  GlobalVariable *CombinedGlobal,
221  const DenseMap<GlobalVariable *, uint64_t> &GlobalLayout);
222  void buildBitSetsFromGlobals(const std::vector<MDString *> &BitSets,
223  const std::vector<GlobalVariable *> &Globals);
224  bool buildBitSets();
225  bool eraseBitSetMetadata();
226 
227  bool doInitialization(Module &M) override;
228  bool runOnModule(Module &M) override;
229 };
230 
231 } // namespace
232 
233 INITIALIZE_PASS_BEGIN(LowerBitSets, "lowerbitsets",
234  "Lower bitset metadata", false, false)
236  "Lower bitset metadata", false, false)
237 char LowerBitSets::ID = 0;
238 
239 ModulePass *llvm::createLowerBitSetsPass() { return new LowerBitSets; }
240 
241 bool LowerBitSets::doInitialization(Module &Mod) {
242  M = &Mod;
243  const DataLayout &DL = Mod.getDataLayout();
244 
245  Triple TargetTriple(M->getTargetTriple());
246  LinkerSubsectionsViaSymbols = TargetTriple.isMacOSX();
247 
248  Int1Ty = Type::getInt1Ty(M->getContext());
249  Int8Ty = Type::getInt8Ty(M->getContext());
250  Int32Ty = Type::getInt32Ty(M->getContext());
251  Int32PtrTy = PointerType::getUnqual(Int32Ty);
252  Int64Ty = Type::getInt64Ty(M->getContext());
253  IntPtrTy = DL.getIntPtrType(M->getContext(), 0);
254 
255  BitSetNM = M->getNamedMetadata("llvm.bitsets");
256 
257  BitSetTestCallSites.clear();
258 
259  return false;
260 }
261 
262 /// Build a bit set for BitSet using the object layouts in
263 /// GlobalLayout.
264 BitSetInfo LowerBitSets::buildBitSet(
265  MDString *BitSet,
266  const DenseMap<GlobalVariable *, uint64_t> &GlobalLayout) {
267  BitSetBuilder BSB;
268 
269  // Compute the byte offset of each element of this bitset.
270  if (BitSetNM) {
271  for (MDNode *Op : BitSetNM->operands()) {
272  if (Op->getOperand(0) != BitSet || !Op->getOperand(1))
273  continue;
274  auto OpGlobal = dyn_cast<GlobalVariable>(
275  cast<ConstantAsMetadata>(Op->getOperand(1))->getValue());
276  if (!OpGlobal)
277  continue;
278  uint64_t Offset =
279  cast<ConstantInt>(cast<ConstantAsMetadata>(Op->getOperand(2))
280  ->getValue())->getZExtValue();
281 
282  Offset += GlobalLayout.find(OpGlobal)->second;
283 
284  BSB.addOffset(Offset);
285  }
286  }
287 
288  return BSB.build();
289 }
290 
291 /// Build a test that bit BitOffset mod sizeof(Bits)*8 is set in
292 /// Bits. This pattern matches to the bt instruction on x86.
294  Value *BitOffset) {
295  auto BitsType = cast<IntegerType>(Bits->getType());
296  unsigned BitWidth = BitsType->getBitWidth();
297 
298  BitOffset = B.CreateZExtOrTrunc(BitOffset, BitsType);
299  Value *BitIndex =
300  B.CreateAnd(BitOffset, ConstantInt::get(BitsType, BitWidth - 1));
301  Value *BitMask = B.CreateShl(ConstantInt::get(BitsType, 1), BitIndex);
302  Value *MaskedBits = B.CreateAnd(Bits, BitMask);
303  return B.CreateICmpNE(MaskedBits, ConstantInt::get(BitsType, 0));
304 }
305 
306 ByteArrayInfo *LowerBitSets::createByteArray(BitSetInfo &BSI) {
307  // Create globals to stand in for byte arrays and masks. These never actually
308  // get initialized, we RAUW and erase them later in allocateByteArrays() once
309  // we know the offset and mask to use.
310  auto ByteArrayGlobal = new GlobalVariable(
311  *M, Int8Ty, /*isConstant=*/true, GlobalValue::PrivateLinkage, nullptr);
312  auto MaskGlobal = new GlobalVariable(
313  *M, Int8Ty, /*isConstant=*/true, GlobalValue::PrivateLinkage, nullptr);
314 
315  ByteArrayInfos.emplace_back();
316  ByteArrayInfo *BAI = &ByteArrayInfos.back();
317 
318  BAI->Bits = BSI.Bits;
319  BAI->BitSize = BSI.BitSize;
320  BAI->ByteArray = ByteArrayGlobal;
321  BAI->Mask = ConstantExpr::getPtrToInt(MaskGlobal, Int8Ty);
322  return BAI;
323 }
324 
325 void LowerBitSets::allocateByteArrays() {
326  std::stable_sort(ByteArrayInfos.begin(), ByteArrayInfos.end(),
327  [](const ByteArrayInfo &BAI1, const ByteArrayInfo &BAI2) {
328  return BAI1.BitSize > BAI2.BitSize;
329  });
330 
331  std::vector<uint64_t> ByteArrayOffsets(ByteArrayInfos.size());
332 
333  ByteArrayBuilder BAB;
334  for (unsigned I = 0; I != ByteArrayInfos.size(); ++I) {
335  ByteArrayInfo *BAI = &ByteArrayInfos[I];
336 
337  uint8_t Mask;
338  BAB.allocate(BAI->Bits, BAI->BitSize, ByteArrayOffsets[I], Mask);
339 
340  BAI->Mask->replaceAllUsesWith(ConstantInt::get(Int8Ty, Mask));
341  cast<GlobalVariable>(BAI->Mask->getOperand(0))->eraseFromParent();
342  }
343 
344  Constant *ByteArrayConst = ConstantDataArray::get(M->getContext(), BAB.Bytes);
345  auto ByteArray =
346  new GlobalVariable(*M, ByteArrayConst->getType(), /*isConstant=*/true,
347  GlobalValue::PrivateLinkage, ByteArrayConst);
348 
349  for (unsigned I = 0; I != ByteArrayInfos.size(); ++I) {
350  ByteArrayInfo *BAI = &ByteArrayInfos[I];
351 
352  Constant *Idxs[] = {ConstantInt::get(IntPtrTy, 0),
353  ConstantInt::get(IntPtrTy, ByteArrayOffsets[I])};
355  ByteArrayConst->getType(), ByteArray, Idxs);
356 
357  // Create an alias instead of RAUW'ing the gep directly. On x86 this ensures
358  // that the pc-relative displacement is folded into the lea instead of the
359  // test instruction getting another displacement.
360  if (LinkerSubsectionsViaSymbols) {
361  BAI->ByteArray->replaceAllUsesWith(GEP);
362  } else {
363  GlobalAlias *Alias =
365  GlobalValue::PrivateLinkage, "bits", GEP, M);
366  BAI->ByteArray->replaceAllUsesWith(Alias);
367  }
368  BAI->ByteArray->eraseFromParent();
369  }
370 
371  ByteArraySizeBits = BAB.BitAllocs[0] + BAB.BitAllocs[1] + BAB.BitAllocs[2] +
372  BAB.BitAllocs[3] + BAB.BitAllocs[4] + BAB.BitAllocs[5] +
373  BAB.BitAllocs[6] + BAB.BitAllocs[7];
374  ByteArraySizeBytes = BAB.Bytes.size();
375 }
376 
377 /// Build a test that bit BitOffset is set in BSI, where
378 /// BitSetGlobal is a global containing the bits in BSI.
379 Value *LowerBitSets::createBitSetTest(IRBuilder<> &B, BitSetInfo &BSI,
380  ByteArrayInfo *&BAI, Value *BitOffset) {
381  if (BSI.BitSize <= 64) {
382  // If the bit set is sufficiently small, we can avoid a load by bit testing
383  // a constant.
384  IntegerType *BitsTy;
385  if (BSI.BitSize <= 32)
386  BitsTy = Int32Ty;
387  else
388  BitsTy = Int64Ty;
389 
390  uint64_t Bits = 0;
391  for (auto Bit : BSI.Bits)
392  Bits |= uint64_t(1) << Bit;
393  Constant *BitsConst = ConstantInt::get(BitsTy, Bits);
394  return createMaskedBitTest(B, BitsConst, BitOffset);
395  } else {
396  if (!BAI) {
397  ++NumByteArraysCreated;
398  BAI = createByteArray(BSI);
399  }
400 
401  Constant *ByteArray = BAI->ByteArray;
402  Type *Ty = BAI->ByteArray->getValueType();
403  if (!LinkerSubsectionsViaSymbols && AvoidReuse) {
404  // Each use of the byte array uses a different alias. This makes the
405  // backend less likely to reuse previously computed byte array addresses,
406  // improving the security of the CFI mechanism based on this pass.
407  ByteArray = GlobalAlias::create(BAI->ByteArray->getType(),
408  GlobalValue::PrivateLinkage, "bits_use",
409  ByteArray, M);
410  }
411 
412  Value *ByteAddr = B.CreateGEP(Ty, ByteArray, BitOffset);
413  Value *Byte = B.CreateLoad(ByteAddr);
414 
415  Value *ByteAndMask = B.CreateAnd(Byte, BAI->Mask);
416  return B.CreateICmpNE(ByteAndMask, ConstantInt::get(Int8Ty, 0));
417  }
418 }
419 
420 /// Lower a llvm.bitset.test call to its implementation. Returns the value to
421 /// replace the call with.
422 Value *LowerBitSets::lowerBitSetCall(
423  CallInst *CI, BitSetInfo &BSI, ByteArrayInfo *&BAI,
424  GlobalVariable *CombinedGlobal,
425  const DenseMap<GlobalVariable *, uint64_t> &GlobalLayout) {
426  Value *Ptr = CI->getArgOperand(0);
427  const DataLayout &DL = M->getDataLayout();
428 
429  if (BSI.containsValue(DL, GlobalLayout, Ptr))
430  return ConstantInt::getTrue(CombinedGlobal->getParent()->getContext());
431 
432  Constant *GlobalAsInt = ConstantExpr::getPtrToInt(CombinedGlobal, IntPtrTy);
433  Constant *OffsetedGlobalAsInt = ConstantExpr::getAdd(
434  GlobalAsInt, ConstantInt::get(IntPtrTy, BSI.ByteOffset));
435 
436  BasicBlock *InitialBB = CI->getParent();
437 
438  IRBuilder<> B(CI);
439 
440  Value *PtrAsInt = B.CreatePtrToInt(Ptr, IntPtrTy);
441 
442  if (BSI.isSingleOffset())
443  return B.CreateICmpEQ(PtrAsInt, OffsetedGlobalAsInt);
444 
445  Value *PtrOffset = B.CreateSub(PtrAsInt, OffsetedGlobalAsInt);
446 
447  Value *BitOffset;
448  if (BSI.AlignLog2 == 0) {
449  BitOffset = PtrOffset;
450  } else {
451  // We need to check that the offset both falls within our range and is
452  // suitably aligned. We can check both properties at the same time by
453  // performing a right rotate by log2(alignment) followed by an integer
454  // comparison against the bitset size. The rotate will move the lower
455  // order bits that need to be zero into the higher order bits of the
456  // result, causing the comparison to fail if they are nonzero. The rotate
457  // also conveniently gives us a bit offset to use during the load from
458  // the bitset.
459  Value *OffsetSHR =
460  B.CreateLShr(PtrOffset, ConstantInt::get(IntPtrTy, BSI.AlignLog2));
461  Value *OffsetSHL = B.CreateShl(
462  PtrOffset,
463  ConstantInt::get(IntPtrTy, DL.getPointerSizeInBits(0) - BSI.AlignLog2));
464  BitOffset = B.CreateOr(OffsetSHR, OffsetSHL);
465  }
466 
467  Constant *BitSizeConst = ConstantInt::get(IntPtrTy, BSI.BitSize);
468  Value *OffsetInRange = B.CreateICmpULT(BitOffset, BitSizeConst);
469 
470  // If the bit set is all ones, testing against it is unnecessary.
471  if (BSI.isAllOnes())
472  return OffsetInRange;
473 
474  TerminatorInst *Term = SplitBlockAndInsertIfThen(OffsetInRange, CI, false);
475  IRBuilder<> ThenB(Term);
476 
477  // Now that we know that the offset is in range and aligned, load the
478  // appropriate bit from the bitset.
479  Value *Bit = createBitSetTest(ThenB, BSI, BAI, BitOffset);
480 
481  // The value we want is 0 if we came directly from the initial block
482  // (having failed the range or alignment checks), or the loaded bit if
483  // we came from the block in which we loaded it.
484  B.SetInsertPoint(CI);
485  PHINode *P = B.CreatePHI(Int1Ty, 2);
486  P->addIncoming(ConstantInt::get(Int1Ty, 0), InitialBB);
487  P->addIncoming(Bit, ThenB.GetInsertBlock());
488  return P;
489 }
490 
491 /// Given a disjoint set of bitsets and globals, layout the globals, build the
492 /// bit sets and lower the llvm.bitset.test calls.
493 void LowerBitSets::buildBitSetsFromGlobals(
494  const std::vector<MDString *> &BitSets,
495  const std::vector<GlobalVariable *> &Globals) {
496  // Build a new global with the combined contents of the referenced globals.
497  std::vector<Constant *> GlobalInits;
498  const DataLayout &DL = M->getDataLayout();
499  for (GlobalVariable *G : Globals) {
500  GlobalInits.push_back(G->getInitializer());
501  uint64_t InitSize = DL.getTypeAllocSize(G->getInitializer()->getType());
502 
503  // Compute the amount of padding required to align the next element to the
504  // next power of 2.
505  uint64_t Padding = NextPowerOf2(InitSize - 1) - InitSize;
506 
507  // Cap at 128 was found experimentally to have a good data/instruction
508  // overhead tradeoff.
509  if (Padding > 128)
510  Padding = RoundUpToAlignment(InitSize, 128) - InitSize;
511 
512  GlobalInits.push_back(
513  ConstantAggregateZero::get(ArrayType::get(Int8Ty, Padding)));
514  }
515  if (!GlobalInits.empty())
516  GlobalInits.pop_back();
517  Constant *NewInit = ConstantStruct::getAnon(M->getContext(), GlobalInits);
518  auto CombinedGlobal =
519  new GlobalVariable(*M, NewInit->getType(), /*isConstant=*/true,
520  GlobalValue::PrivateLinkage, NewInit);
521 
522  const StructLayout *CombinedGlobalLayout =
523  DL.getStructLayout(cast<StructType>(NewInit->getType()));
524 
525  // Compute the offsets of the original globals within the new global.
527  for (unsigned I = 0; I != Globals.size(); ++I)
528  // Multiply by 2 to account for padding elements.
529  GlobalLayout[Globals[I]] = CombinedGlobalLayout->getElementOffset(I * 2);
530 
531  // For each bitset in this disjoint set...
532  for (MDString *BS : BitSets) {
533  // Build the bitset.
534  BitSetInfo BSI = buildBitSet(BS, GlobalLayout);
535 
536  ByteArrayInfo *BAI = 0;
537 
538  // Lower each call to llvm.bitset.test for this bitset.
539  for (CallInst *CI : BitSetTestCallSites[BS]) {
540  ++NumBitSetCallsLowered;
541  Value *Lowered = lowerBitSetCall(CI, BSI, BAI, CombinedGlobal, GlobalLayout);
542  CI->replaceAllUsesWith(Lowered);
543  CI->eraseFromParent();
544  }
545  }
546 
547  // Build aliases pointing to offsets into the combined global for each
548  // global from which we built the combined global, and replace references
549  // to the original globals with references to the aliases.
550  for (unsigned I = 0; I != Globals.size(); ++I) {
551  // Multiply by 2 to account for padding elements.
552  Constant *CombinedGlobalIdxs[] = {ConstantInt::get(Int32Ty, 0),
553  ConstantInt::get(Int32Ty, I * 2)};
554  Constant *CombinedGlobalElemPtr = ConstantExpr::getGetElementPtr(
555  NewInit->getType(), CombinedGlobal, CombinedGlobalIdxs);
556  if (LinkerSubsectionsViaSymbols) {
557  Globals[I]->replaceAllUsesWith(CombinedGlobalElemPtr);
558  } else {
559  GlobalAlias *GAlias =
560  GlobalAlias::create(Globals[I]->getType(), Globals[I]->getLinkage(),
561  "", CombinedGlobalElemPtr, M);
562  GAlias->takeName(Globals[I]);
563  Globals[I]->replaceAllUsesWith(GAlias);
564  }
565  Globals[I]->eraseFromParent();
566  }
567 }
568 
569 /// Lower all bit sets in this module.
570 bool LowerBitSets::buildBitSets() {
571  Function *BitSetTestFunc =
572  M->getFunction(Intrinsic::getName(Intrinsic::bitset_test));
573  if (!BitSetTestFunc)
574  return false;
575 
576  // Equivalence class set containing bitsets and the globals they reference.
577  // This is used to partition the set of bitsets in the module into disjoint
578  // sets.
580  GlobalClassesTy;
581  GlobalClassesTy GlobalClasses;
582 
583  for (const Use &U : BitSetTestFunc->uses()) {
584  auto CI = cast<CallInst>(U.getUser());
585 
586  auto BitSetMDVal = dyn_cast<MetadataAsValue>(CI->getArgOperand(1));
587  if (!BitSetMDVal || !isa<MDString>(BitSetMDVal->getMetadata()))
589  "Second argument of llvm.bitset.test must be metadata string");
590  auto BitSet = cast<MDString>(BitSetMDVal->getMetadata());
591 
592  // Add the call site to the list of call sites for this bit set. We also use
593  // BitSetTestCallSites to keep track of whether we have seen this bit set
594  // before. If we have, we don't need to re-add the referenced globals to the
595  // equivalence class.
596  std::pair<DenseMap<MDString *, std::vector<CallInst *>>::iterator,
597  bool> Ins =
598  BitSetTestCallSites.insert(
599  std::make_pair(BitSet, std::vector<CallInst *>()));
600  Ins.first->second.push_back(CI);
601  if (!Ins.second)
602  continue;
603 
604  // Add the bitset to the equivalence class.
605  GlobalClassesTy::iterator GCI = GlobalClasses.insert(BitSet);
606  GlobalClassesTy::member_iterator CurSet = GlobalClasses.findLeader(GCI);
607 
608  if (!BitSetNM)
609  continue;
610 
611  // Verify the bitset metadata and add the referenced globals to the bitset's
612  // equivalence class.
613  for (MDNode *Op : BitSetNM->operands()) {
614  if (Op->getNumOperands() != 3)
616  "All operands of llvm.bitsets metadata must have 3 elements");
617 
618  if (Op->getOperand(0) != BitSet || !Op->getOperand(1))
619  continue;
620 
621  auto OpConstMD = dyn_cast<ConstantAsMetadata>(Op->getOperand(1));
622  if (!OpConstMD)
623  report_fatal_error("Bit set element must be a constant");
624  auto OpGlobal = dyn_cast<GlobalVariable>(OpConstMD->getValue());
625  if (!OpGlobal)
626  continue;
627 
628  auto OffsetConstMD = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
629  if (!OffsetConstMD)
630  report_fatal_error("Bit set element offset must be a constant");
631  auto OffsetInt = dyn_cast<ConstantInt>(OffsetConstMD->getValue());
632  if (!OffsetInt)
634  "Bit set element offset must be an integer constant");
635 
636  CurSet = GlobalClasses.unionSets(
637  CurSet, GlobalClasses.findLeader(GlobalClasses.insert(OpGlobal)));
638  }
639  }
640 
641  if (GlobalClasses.empty())
642  return false;
643 
644  // For each disjoint set we found...
645  for (GlobalClassesTy::iterator I = GlobalClasses.begin(),
646  E = GlobalClasses.end();
647  I != E; ++I) {
648  if (!I->isLeader()) continue;
649 
650  ++NumBitSetDisjointSets;
651 
652  // Build the list of bitsets and referenced globals in this disjoint set.
653  std::vector<MDString *> BitSets;
654  std::vector<GlobalVariable *> Globals;
657  for (GlobalClassesTy::member_iterator MI = GlobalClasses.member_begin(I);
658  MI != GlobalClasses.member_end(); ++MI) {
659  if ((*MI).is<MDString *>()) {
660  BitSetIndices[MI->get<MDString *>()] = BitSets.size();
661  BitSets.push_back(MI->get<MDString *>());
662  } else {
663  GlobalIndices[MI->get<GlobalVariable *>()] = Globals.size();
664  Globals.push_back(MI->get<GlobalVariable *>());
665  }
666  }
667 
668  // For each bitset, build a set of indices that refer to globals referenced
669  // by the bitset.
670  std::vector<std::set<uint64_t>> BitSetMembers(BitSets.size());
671  if (BitSetNM) {
672  for (MDNode *Op : BitSetNM->operands()) {
673  // Op = { bitset name, global, offset }
674  if (!Op->getOperand(1))
675  continue;
676  auto I = BitSetIndices.find(cast<MDString>(Op->getOperand(0)));
677  if (I == BitSetIndices.end())
678  continue;
679 
680  auto OpGlobal = dyn_cast<GlobalVariable>(
681  cast<ConstantAsMetadata>(Op->getOperand(1))->getValue());
682  if (!OpGlobal)
683  continue;
684  BitSetMembers[I->second].insert(GlobalIndices[OpGlobal]);
685  }
686  }
687 
688  // Order the sets of indices by size. The GlobalLayoutBuilder works best
689  // when given small index sets first.
690  std::stable_sort(
691  BitSetMembers.begin(), BitSetMembers.end(),
692  [](const std::set<uint64_t> &O1, const std::set<uint64_t> &O2) {
693  return O1.size() < O2.size();
694  });
695 
696  // Create a GlobalLayoutBuilder and provide it with index sets as layout
697  // fragments. The GlobalLayoutBuilder tries to lay out members of fragments
698  // as close together as possible.
699  GlobalLayoutBuilder GLB(Globals.size());
700  for (auto &&MemSet : BitSetMembers)
701  GLB.addFragment(MemSet);
702 
703  // Build a vector of globals with the computed layout.
704  std::vector<GlobalVariable *> OrderedGlobals(Globals.size());
705  auto OGI = OrderedGlobals.begin();
706  for (auto &&F : GLB.Fragments)
707  for (auto &&Offset : F)
708  *OGI++ = Globals[Offset];
709 
710  // Order bitsets by name for determinism.
711  std::sort(BitSets.begin(), BitSets.end(), [](MDString *S1, MDString *S2) {
712  return S1->getString() < S2->getString();
713  });
714 
715  // Build the bitsets from this disjoint set.
716  buildBitSetsFromGlobals(BitSets, OrderedGlobals);
717  }
718 
719  allocateByteArrays();
720 
721  return true;
722 }
723 
724 bool LowerBitSets::eraseBitSetMetadata() {
725  if (!BitSetNM)
726  return false;
727 
728  M->eraseNamedMetadata(BitSetNM);
729  return true;
730 }
731 
732 bool LowerBitSets::runOnModule(Module &M) {
733  bool Changed = buildBitSets();
734  Changed |= eraseBitSetMetadata();
735  return Changed;
736 }
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:842
Value * CreateGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Definition: IRBuilder.h:1032
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
Definition: Instruction.cpp:70
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
BitSetInfo build()
iterator_range< use_iterator > uses()
Definition: Value.h:283
static IntegerType * getInt1Ty(LLVMContext &C)
Definition: Type.cpp:236
LoadInst * CreateLoad(Value *Ptr, const char *Name)
Definition: IRBuilder.h:973
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
This class is used to build a byte array containing overlapping bit sets.
Definition: LowerBitSets.h:174
STATISTIC(NumFunctions,"Total number of functions")
bool isAllOnes() const
Definition: LowerBitSets.h:51
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:114
static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)
getAnon - Return an anonymous struct that has the specified elements.
Definition: Constants.h:418
uint64_t BitSize
Definition: LowerBitSets.h:40
static ConstantAggregateZero * get(Type *Ty)
Definition: Constants.cpp:1377
CallInst - This class represents a function call, abstracting a target machine's calling convention...
TerminatorInst * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DominatorTree *DT=nullptr)
SplitBlockAndInsertIfThen - Split the containing block at the specified instruction - everything befo...
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, bool InBounds=false, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
Definition: Constants.h:1092
std::vector< uint8_t > Bytes
The byte array built so far.
Definition: LowerBitSets.h:176
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:48
bool containsValue(const DataLayout &DL, const DenseMap< GlobalVariable *, uint64_t > &GlobalLayout, Value *V, uint64_t COffset=0) const
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1366
Metadata node.
Definition: Metadata.h:740
F(f)
lowerbitsets
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:240
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
Definition: Function.cpp:822
Hexagon Common GEP
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1354
void allocate(const std::set< uint64_t > &Bits, uint64_t BitSize, uint64_t &AllocByteOffset, uint8_t &AllocMask)
Allocate BitSize bits in the byte array where Bits contains the bits to set.
SmallVector< uint64_t, 16 > Offsets
Definition: LowerBitSets.h:63
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:726
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
static Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2258
bool isMacOSX() const
isMacOSX - Is this a Mac OS X triple.
Definition: Triple.h:394
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:1462
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition: DataLayout.h:475
A tuple of MDNodes.
Definition: Metadata.h:1127
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:551
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with array type with an element count and element type matchin...
Definition: Constants.cpp:2552
A Use represents the edge between a Value definition and its users.
Definition: Use.h:69
The returned value is undefined.
Definition: MathExtras.h:35
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:75
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:517
INITIALIZE_PASS_BEGIN(LowerBitSets,"lowerbitsets","Lower bitset metadata", false, false) INITIALIZE_PASS_END(LowerBitSets
#define G(x, y, z)
Definition: MD5.cpp:52
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:878
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:894
std::vector< std::vector< uint64_t > > Fragments
The computed layout.
Definition: LowerBitSets.h:129
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:351
void addFragment(const std::set< uint64_t > &F)
Add F to the layout while trying to keep its indices contiguous.
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:256
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Definition: IRBuilder.h:85
std::vector< uint64_t > FragmentMap
Mapping from object index to fragment index.
Definition: LowerBitSets.h:132
Lower bitset metadata
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
static Constant * getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList)
Create an "inbounds" getelementptr.
Definition: Constants.h:1115
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:35
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:109
LLVM Basic Block Representation.
Definition: BasicBlock.h:65
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
uint64_t BitAllocs[BitsPerByte]
The number of bytes allocated so far for each of the bits.
Definition: LowerBitSets.h:181
static GlobalAlias * create(PointerType *Ty, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
Definition: Globals.cpp:243
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1245
This is an important base class in LLVM.
Definition: Constant.h:41
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ModulePass * createLowerBitSetsPass()
This pass lowers bitset metadata and the llvm.bitset.test intrinsic to bitsets.
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
Class to represent integer types.
Definition: DerivedTypes.h:37
Metadata wrapper in the Value hierarchy.
Definition: Metadata.h:172
bool isSingleOffset() const
Definition: LowerBitSets.h:47
uint64_t NextPowerOf2(uint64_t A)
NextPowerOf2 - Returns the next power of two (in 64-bits) that is strictly greater than A...
Definition: MathExtras.h:582
std::set< uint64_t > Bits
Definition: LowerBitSets.h:34
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
Definition: DataLayout.cpp:694
StringRef getString() const
Definition: Metadata.cpp:375
static PointerType * getUnqual(Type *ElementType)
PointerType::getUnqual - This constructs a pointer to an object of the specified type in the generic ...
Definition: DerivedTypes.h:460
This is the shared class of boolean and integer constants.
Definition: Constants.h:47
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:388
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:582
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:530
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
Class for arbitrary precision integers.
Definition: APInt.h:73
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
uint64_t RoundUpToAlignment(uint64_t Value, uint64_t Align)
Returns the next integer (mod 2**64) that is greater than or equal to Value and is a multiple of Alig...
Definition: MathExtras.h:609
This class implements a layout algorithm for globals referenced by bit sets that tries to keep member...
Definition: LowerBitSets.h:126
void initializeLowerBitSetsPass(PassRegistry &)
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:823
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:372
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Definition: IRBuilder.h:1200
static cl::opt< bool > AvoidReuse("lowerbitsets-avoid-reuse", cl::desc("Try to avoid reuse of byte array addresses using aliases"), cl::Hidden, cl::init(true))
Lower bitset false
std::string getName(ID id, ArrayRef< Type * > Tys=None)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:500
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:239
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1809
#define I(x, y, z)
Definition: MD5.cpp:54
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:236
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:329
static ArrayType * get(Type *ElementType, uint64_t NumElements)
ArrayType::get - This static method is the primary way to construct an ArrayType. ...
Definition: Type.cpp:686
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1357
void addOffset(uint64_t Offset)
Definition: LowerBitSets.h:68
op_range operands() const
Definition: Metadata.h:934
uint64_t ByteOffset
Definition: LowerBitSets.h:37
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:365
LLVM Value Representation.
Definition: Value.h:69
static Value * createMaskedBitTest(IRBuilder<> &B, Value *Bits, Value *BitOffset)
Build a test that bit BitOffset mod sizeof(Bits)*8 is set in Bits.
A single uniqued string.
Definition: Metadata.h:508
unsigned AlignLog2
Definition: LowerBitSets.h:45
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:237
bool containsGlobalOffset(uint64_t Offset) const
const BasicBlock * getParent() const
Definition: Instruction.h:72
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:265