LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - AMDGPUOpenCLImageTypeLoweringPass.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 157 172 91.3 %
Date: 2017-09-14 15:23:50 Functions: 16 17 94.1 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- AMDGPUOpenCLImageTypeLoweringPass.cpp ------------------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// This pass resolves calls to OpenCL image attribute, image resource ID and
      12             : /// sampler resource ID getter functions.
      13             : ///
      14             : /// Image attributes (size and format) are expected to be passed to the kernel
      15             : /// as kernel arguments immediately following the image argument itself,
      16             : /// therefore this pass adds image size and format arguments to the kernel
      17             : /// functions in the module. The kernel functions with image arguments are
      18             : /// re-created using the new signature. The new arguments are added to the
      19             : /// kernel metadata with kernel_arg_type set to "image_size" or "image_format".
      20             : /// Note: this pass may invalidate pointers to functions.
      21             : ///
      22             : /// Resource IDs of read-only images, write-only images and samplers are
      23             : /// defined to be their index among the kernel arguments of the same
      24             : /// type and access qualifier.
      25             : //
      26             : //===----------------------------------------------------------------------===//
      27             : 
      28             : #include "AMDGPU.h"
      29             : #include "llvm/ADT/SmallVector.h"
      30             : #include "llvm/ADT/StringRef.h"
      31             : #include "llvm/ADT/Twine.h"
      32             : #include "llvm/IR/Argument.h"
      33             : #include "llvm/IR/DerivedTypes.h"
      34             : #include "llvm/IR/Constants.h"
      35             : #include "llvm/IR/Function.h"
      36             : #include "llvm/IR/Instruction.h"
      37             : #include "llvm/IR/Instructions.h"
      38             : #include "llvm/IR/Metadata.h"
      39             : #include "llvm/IR/Module.h"
      40             : #include "llvm/IR/Type.h"
      41             : #include "llvm/IR/Use.h"
      42             : #include "llvm/IR/User.h"
      43             : #include "llvm/Pass.h"
      44             : #include "llvm/Support/Casting.h"
      45             : #include "llvm/Support/ErrorHandling.h"
      46             : #include "llvm/Transforms/Utils/Cloning.h"
      47             : #include "llvm/Transforms/Utils/ValueMapper.h"
      48             : #include <cassert>
      49             : #include <cstddef>
      50             : #include <cstdint>
      51             : #include <tuple>
      52             : 
      53             : using namespace llvm;
      54             : 
      55       72306 : static StringRef GetImageSizeFunc =         "llvm.OpenCL.image.get.size";
      56       72306 : static StringRef GetImageFormatFunc =       "llvm.OpenCL.image.get.format";
      57       72306 : static StringRef GetImageResourceIDFunc =   "llvm.OpenCL.image.get.resource.id";
      58       72306 : static StringRef GetSamplerResourceIDFunc =
      59             :     "llvm.OpenCL.sampler.get.resource.id";
      60             : 
      61       72306 : static StringRef ImageSizeArgMDType =   "__llvm_image_size";
      62       72306 : static StringRef ImageFormatArgMDType = "__llvm_image_format";
      63             : 
      64       72306 : static StringRef KernelsMDNodeName = "opencl.kernels";
      65             : static StringRef KernelArgMDNodeNames[] = {
      66             :   "kernel_arg_addr_space",
      67             :   "kernel_arg_access_qual",
      68             :   "kernel_arg_type",
      69             :   "kernel_arg_base_type",
      70      433836 :   "kernel_arg_type_qual"};
      71             : static const unsigned NumKernelArgMDNodes = 5;
      72             : 
      73             : namespace {
      74             : 
      75             : using MDVector = SmallVector<Metadata *, 8>;
      76         702 : struct KernelArgMD {
      77             :   MDVector ArgVector[NumKernelArgMDNodes];
      78             : };
      79             : 
      80             : } // end anonymous namespace
      81             : 
      82             : static inline bool
      83         288 : IsImageType(StringRef TypeString) {
      84         783 :   return TypeString == "image2d_t" || TypeString == "image3d_t";
      85             : }
      86             : 
      87             : static inline bool
      88             : IsSamplerType(StringRef TypeString) {
      89          53 :   return TypeString == "sampler_t";
      90             : }
      91             : 
      92             : static Function *
      93          66 : GetFunctionFromMDNode(MDNode *Node) {
      94          66 :   if (!Node)
      95             :     return nullptr;
      96             : 
      97          66 :   size_t NumOps = Node->getNumOperands();
      98          66 :   if (NumOps != NumKernelArgMDNodes + 1)
      99             :     return nullptr;
     100             : 
     101          82 :   auto F = mdconst::dyn_extract<Function>(Node->getOperand(0));
     102             :   if (!F)
     103             :     return nullptr;
     104             : 
     105             :   // Sanity checks.
     106          41 :   size_t ExpectNumArgNodeOps = F->arg_size() + 1;
     107         242 :   for (size_t i = 0; i < NumKernelArgMDNodes; ++i) {
     108         609 :     MDNode *ArgNode = dyn_cast_or_null<MDNode>(Node->getOperand(i + 1));
     109         203 :     if (ArgNode->getNumOperands() != ExpectNumArgNodeOps)
     110             :       return nullptr;
     111         406 :     if (!ArgNode->getOperand(0))
     112             :       return nullptr;
     113             : 
     114             :     // FIXME: It should be possible to do image lowering when some metadata
     115             :     // args missing or not in the expected order.
     116         406 :     MDString *StringNode = dyn_cast<MDString>(ArgNode->getOperand(0));
     117         203 :     if (!StringNode || StringNode->getString() != KernelArgMDNodeNames[i])
     118             :       return nullptr;
     119             :   }
     120             : 
     121             :   return F;
     122             : }
     123             : 
     124             : static StringRef
     125          55 : AccessQualFromMD(MDNode *KernelMDNode, unsigned ArgIdx) {
     126         110 :   MDNode *ArgAQNode = cast<MDNode>(KernelMDNode->getOperand(2));
     127         165 :   return cast<MDString>(ArgAQNode->getOperand(ArgIdx + 1))->getString();
     128             : }
     129             : 
     130             : static StringRef
     131         288 : ArgTypeFromMD(MDNode *KernelMDNode, unsigned ArgIdx) {
     132         576 :   MDNode *ArgTypeNode = cast<MDNode>(KernelMDNode->getOperand(3));
     133         864 :   return cast<MDString>(ArgTypeNode->getOperand(ArgIdx + 1))->getString();
     134             : }
     135             : 
     136             : static MDVector
     137         140 : GetArgMD(MDNode *KernelMDNode, unsigned OpIdx) {
     138         140 :   MDVector Res;
     139         840 :   for (unsigned i = 0; i < NumKernelArgMDNodes; ++i) {
     140        2100 :     MDNode *Node = cast<MDNode>(KernelMDNode->getOperand(i + 1));
     141        2100 :     Res.push_back(Node->getOperand(OpIdx));
     142             :   }
     143         140 :   return Res;
     144             : }
     145             : 
     146             : static void
     147             : PushArgMD(KernelArgMD &MD, const MDVector &V) {
     148             :   assert(V.size() == NumKernelArgMDNodes);
     149        2750 :   for (unsigned i = 0; i < NumKernelArgMDNodes; ++i) {
     150        2500 :     MD.ArgVector[i].push_back(V[i]);
     151             :   }
     152             : }
     153             : 
     154             : namespace {
     155             : 
     156        5109 : class AMDGPUOpenCLImageTypeLoweringPass : public ModulePass {
     157             :   static char ID;
     158             : 
     159             :   LLVMContext *Context;
     160             :   Type *Int32Type;
     161             :   Type *ImageSizeType;
     162             :   Type *ImageFormatType;
     163             :   SmallVector<Instruction *, 4> InstsToErase;
     164             : 
     165          55 :   bool replaceImageUses(Argument &ImageArg, uint32_t ResourceID,
     166             :                         Argument &ImageSizeArg,
     167             :                         Argument &ImageFormatArg) {
     168          55 :     bool Modified = false;
     169             : 
     170         195 :     for (auto &Use : ImageArg.uses()) {
     171          60 :       auto Inst = dyn_cast<CallInst>(Use.getUser());
     172           0 :       if (!Inst) {
     173           0 :         continue;
     174             :       }
     175             : 
     176          30 :       Function *F = Inst->getCalledFunction();
     177           0 :       if (!F)
     178           0 :         continue;
     179             : 
     180          30 :       Value *Replacement = nullptr;
     181          30 :       StringRef Name = F->getName();
     182             :       if (Name.startswith(GetImageResourceIDFunc)) {
     183          20 :         Replacement = ConstantInt::get(Int32Type, ResourceID);
     184             :       } else if (Name.startswith(GetImageSizeFunc)) {
     185           6 :         Replacement = &ImageSizeArg;
     186           0 :       } else if (Name.startswith(GetImageFormatFunc)) {
     187           4 :         Replacement = &ImageFormatArg;
     188             :       } else {
     189           0 :         continue;
     190             :       }
     191             : 
     192          30 :       Inst->replaceAllUsesWith(Replacement);
     193          30 :       InstsToErase.push_back(Inst);
     194          30 :       Modified = true;
     195             :     }
     196             : 
     197          55 :     return Modified;
     198             :   }
     199             : 
     200           7 :   bool replaceSamplerUses(Argument &SamplerArg, uint32_t ResourceID) {
     201           7 :     bool Modified = false;
     202             : 
     203          24 :     for (const auto &Use : SamplerArg.uses()) {
     204           6 :       auto Inst = dyn_cast<CallInst>(Use.getUser());
     205           0 :       if (!Inst) {
     206           0 :         continue;
     207             :       }
     208             : 
     209           3 :       Function *F = Inst->getCalledFunction();
     210           0 :       if (!F)
     211           0 :         continue;
     212             : 
     213           3 :       Value *Replacement = nullptr;
     214           3 :       StringRef Name = F->getName();
     215           0 :       if (Name == GetSamplerResourceIDFunc) {
     216           3 :         Replacement = ConstantInt::get(Int32Type, ResourceID);
     217             :       } else {
     218           0 :         continue;
     219             :       }
     220             : 
     221           3 :       Inst->replaceAllUsesWith(Replacement);
     222           3 :       InstsToErase.push_back(Inst);
     223           3 :       Modified = true;
     224             :     }
     225             : 
     226           7 :     return Modified;
     227             :   }
     228             : 
     229          39 :   bool replaceImageAndSamplerUses(Function *F, MDNode *KernelMDNode) {
     230          39 :     uint32_t NumReadOnlyImageArgs = 0;
     231          39 :     uint32_t NumWriteOnlyImageArgs = 0;
     232          39 :     uint32_t NumSamplerArgs = 0;
     233             : 
     234          39 :     bool Modified = false;
     235          78 :     InstsToErase.clear();
     236         280 :     for (auto ArgI = F->arg_begin(); ArgI != F->arg_end(); ++ArgI) {
     237         101 :       Argument &Arg = *ArgI;
     238         101 :       StringRef Type = ArgTypeFromMD(KernelMDNode, Arg.getArgNo());
     239             : 
     240             :       // Handle image types.
     241         101 :       if (IsImageType(Type)) {
     242          55 :         StringRef AccessQual = AccessQualFromMD(KernelMDNode, Arg.getArgNo());
     243             :         uint32_t ResourceID;
     244          87 :         if (AccessQual == "read_only") {
     245          32 :           ResourceID = NumReadOnlyImageArgs++;
     246          46 :         } else if (AccessQual == "write_only") {
     247          23 :           ResourceID = NumWriteOnlyImageArgs++;
     248             :         } else {
     249           0 :           llvm_unreachable("Wrong image access qualifier.");
     250             :         }
     251             : 
     252          55 :         Argument &SizeArg = *(++ArgI);
     253          55 :         Argument &FormatArg = *(++ArgI);
     254          55 :         Modified |= replaceImageUses(Arg, ResourceID, SizeArg, FormatArg);
     255             : 
     256             :       // Handle sampler type.
     257           7 :       } else if (IsSamplerType(Type)) {
     258           7 :         uint32_t ResourceID = NumSamplerArgs++;
     259           7 :         Modified |= replaceSamplerUses(Arg, ResourceID);
     260             :       }
     261             :     }
     262         177 :     for (unsigned i = 0; i < InstsToErase.size(); ++i) {
     263          66 :       InstsToErase[i]->eraseFromParent();
     264             :     }
     265             : 
     266          39 :     return Modified;
     267             :   }
     268             : 
     269             :   std::tuple<Function *, MDNode *>
     270          39 :   addImplicitArgs(Function *F, MDNode *KernelMDNode) {
     271          39 :     bool Modified = false;
     272             : 
     273          39 :     FunctionType *FT = F->getFunctionType();
     274          78 :     SmallVector<Type *, 8> ArgTypes;
     275             : 
     276             :     // Metadata operands for new MDNode.
     277          78 :     KernelArgMD NewArgMDs;
     278         117 :     PushArgMD(NewArgMDs, GetArgMD(KernelMDNode, 0));
     279             : 
     280             :     // Add implicit arguments to the signature.
     281         179 :     for (unsigned i = 0; i < FT->getNumParams(); ++i) {
     282         202 :       ArgTypes.push_back(FT->getParamType(i));
     283         156 :       MDVector ArgMD = GetArgMD(KernelMDNode, i + 1);
     284         101 :       PushArgMD(NewArgMDs, ArgMD);
     285             : 
     286         101 :       if (!IsImageType(ArgTypeFromMD(KernelMDNode, i)))
     287          46 :         continue;
     288             : 
     289             :       // Add size implicit argument.
     290          55 :       ArgTypes.push_back(ImageSizeType);
     291         165 :       ArgMD[2] = ArgMD[3] = MDString::get(*Context, ImageSizeArgMDType);
     292          55 :       PushArgMD(NewArgMDs, ArgMD);
     293             : 
     294             :       // Add format implicit argument.
     295          55 :       ArgTypes.push_back(ImageFormatType);
     296         165 :       ArgMD[2] = ArgMD[3] = MDString::get(*Context, ImageFormatArgMDType);
     297          55 :       PushArgMD(NewArgMDs, ArgMD);
     298             : 
     299          55 :       Modified = true;
     300             :     }
     301          39 :     if (!Modified) {
     302          27 :       return std::make_tuple(nullptr, nullptr);
     303             :     }
     304             : 
     305             :     // Create function with new signature and clone the old body into it.
     306          60 :     auto NewFT = FunctionType::get(FT->getReturnType(), ArgTypes, false);
     307         120 :     auto NewF = Function::Create(NewFT, F->getLinkage(), F->getName());
     308          60 :     ValueToValueMapTy VMap;
     309          60 :     auto NewFArgIt = NewF->arg_begin();
     310         116 :     for (auto &Arg: F->args()) {
     311          86 :       auto ArgName = Arg.getName();
     312         172 :       NewFArgIt->setName(ArgName);
     313         172 :       VMap[&Arg] = &(*NewFArgIt++);
     314          86 :       if (IsImageType(ArgTypeFromMD(KernelMDNode, Arg.getArgNo()))) {
     315         220 :         (NewFArgIt++)->setName(Twine("__size_") + ArgName);
     316         220 :         (NewFArgIt++)->setName(Twine("__format_") + ArgName);
     317             :       }
     318             :     }
     319          60 :     SmallVector<ReturnInst*, 8> Returns;
     320          30 :     CloneFunctionInto(NewF, F, VMap, /*ModuleLevelChanges=*/false, Returns);
     321             : 
     322             :     // Build new MDNode.
     323          60 :     SmallVector<Metadata *, 6> KernelMDArgs;
     324          60 :     KernelMDArgs.push_back(ConstantAsMetadata::get(NewF));
     325         180 :     for (unsigned i = 0; i < NumKernelArgMDNodes; ++i)
     326         450 :       KernelMDArgs.push_back(MDNode::get(*Context, NewArgMDs.ArgVector[i]));
     327          60 :     MDNode *NewMDNode = MDNode::get(*Context, KernelMDArgs);
     328             : 
     329          60 :     return std::make_tuple(NewF, NewMDNode);
     330             :   }
     331             : 
     332        1703 :   bool transformKernels(Module &M) {
     333        1703 :     NamedMDNode *KernelsMDNode = M.getNamedMetadata(KernelsMDNodeName);
     334        1703 :     if (!KernelsMDNode)
     335             :       return false;
     336             : 
     337             :     bool Modified = false;
     338         147 :     for (unsigned i = 0; i < KernelsMDNode->getNumOperands(); ++i) {
     339          66 :       MDNode *KernelMDNode = KernelsMDNode->getOperand(i);
     340          66 :       Function *F = GetFunctionFromMDNode(KernelMDNode);
     341          66 :       if (!F)
     342          27 :         continue;
     343             : 
     344             :       Function *NewF;
     345             :       MDNode *NewMDNode;
     346         117 :       std::tie(NewF, NewMDNode) = addImplicitArgs(F, KernelMDNode);
     347          39 :       if (NewF) {
     348             :         // Replace old function and metadata with new ones.
     349          30 :         F->eraseFromParent();
     350          60 :         M.getFunctionList().push_back(NewF);
     351          60 :         M.getOrInsertFunction(NewF->getName(), NewF->getFunctionType(),
     352             :                               NewF->getAttributes());
     353          30 :         KernelsMDNode->setOperand(i, NewMDNode);
     354             : 
     355          30 :         F = NewF;
     356          30 :         KernelMDNode = NewMDNode;
     357          30 :         Modified = true;
     358             :       }
     359             : 
     360          39 :       Modified |= replaceImageAndSamplerUses(F, KernelMDNode);
     361             :     }
     362             : 
     363             :     return Modified;
     364             :   }
     365             : 
     366             : public:
     367        5136 :   AMDGPUOpenCLImageTypeLoweringPass() : ModulePass(ID) {}
     368             : 
     369        1703 :   bool runOnModule(Module &M) override {
     370        1703 :     Context = &M.getContext();
     371        1703 :     Int32Type = Type::getInt32Ty(M.getContext());
     372        1703 :     ImageSizeType = ArrayType::get(Int32Type, 3);
     373        1703 :     ImageFormatType = ArrayType::get(Int32Type, 2);
     374             : 
     375        1703 :     return transformKernels(M);
     376             :   }
     377             : 
     378           0 :   StringRef getPassName() const override {
     379           0 :     return "AMDGPU OpenCL Image Type Pass";
     380             :   }
     381             : };
     382             : 
     383             : } // end anonymous namespace
     384             : 
     385             : char AMDGPUOpenCLImageTypeLoweringPass::ID = 0;
     386             : 
     387        1712 : ModulePass *llvm::createAMDGPUOpenCLImageTypeLoweringPass() {
     388        3424 :   return new AMDGPUOpenCLImageTypeLoweringPass();
     389      216918 : }

Generated by: LCOV version 1.13