AMDGPUOpenCLEnqueuedBlockLowering.cpp

Go to the documentation of this file.

//===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// \file
// This post-linking pass replaces the function pointer of enqueued
// block kernel with a global variable (runtime handle) and adds
// "runtime-handle" attribute to the enqueued block kernel.
//
// In LLVM CodeGen the runtime-handle metadata will be translated to
// RuntimeHandle metadata in code object. Runtime allocates a global buffer
// for each kernel with RuntimeHandle metadata and saves the kernel address
// required for the AQL packet into the buffer. __enqueue_kernel function
// in device library knows that the invoke function pointer in the block
// literal is actually runtime handle and loads the kernel address from it
// and put it into AQL packet for dispatching.
//
// This cannot be done in FE since FE cannot create a unique global variable
// with external linkage across LLVM modules. The global variable with internal
// linkage does not work since optimization passes will try to replace loads
// of the global variable with its initialization value.
//
// It also identifies the kernels directly or indirectly enqueues kernels
// and adds "calls-enqueue-kernel" function attribute to them, which will
// be used to determine whether to emit runtime metadata for the kernel
// enqueue related hidden kernel arguments.
//
//===----------------------------------------------------------------------===//
 
#include "AMDGPUOpenCLEnqueuedBlockLowering.h"
#include "AMDGPU.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Mangler.h"
#include "llvm/IR/Module.h"
#include "llvm/Pass.h"
#include "llvm/Support/Debug.h"
 
#define DEBUG_TYPE "amdgpu-lower-enqueued-block"
 
using namespace llvm;
 
namespace {
 
/// Lower enqueued blocks.
class AMDGPUOpenCLEnqueuedBlockLowering {
public:
  bool run(Module &M);
};
 
class AMDGPUOpenCLEnqueuedBlockLoweringLegacy : public ModulePass {
public:
  static char ID;
 
  explicit AMDGPUOpenCLEnqueuedBlockLoweringLegacy() : ModulePass(ID) {}
 
private:
  bool runOnModule(Module &M) override;
};
 
} // end anonymous namespace
 
char AMDGPUOpenCLEnqueuedBlockLoweringLegacy::ID = 0;
 
char &llvm::AMDGPUOpenCLEnqueuedBlockLoweringLegacyID =
    AMDGPUOpenCLEnqueuedBlockLoweringLegacy::ID;
 
INITIALIZE_PASS(AMDGPUOpenCLEnqueuedBlockLoweringLegacy, DEBUG_TYPE,
                "Lower OpenCL enqueued blocks", false, false)
 
ModulePass *llvm::createAMDGPUOpenCLEnqueuedBlockLoweringLegacyPass() {
  return new AMDGPUOpenCLEnqueuedBlockLoweringLegacy();
}
 
bool AMDGPUOpenCLEnqueuedBlockLoweringLegacy::runOnModule(Module &M) {
  AMDGPUOpenCLEnqueuedBlockLowering Impl;
  return Impl.run(M);
}
 
PreservedAnalyses
AMDGPUOpenCLEnqueuedBlockLoweringPass::run(Module &M, ModuleAnalysisManager &) {
  AMDGPUOpenCLEnqueuedBlockLowering Impl;
  if (Impl.run(M))
    return PreservedAnalyses::none();
  return PreservedAnalyses::all();
}
 
bool AMDGPUOpenCLEnqueuedBlockLowering::run(Module &M) {
  DenseSet<Function *> Callers;
  auto &C = M.getContext();
  bool Changed = false;
 
  // ptr kernel_object, i32 private_segment_size, i32 group_segment_size
  StructType *HandleTy = nullptr;
 
  for (auto &F : M.functions()) {
    if (F.hasFnAttribute("enqueued-block")) {
      if (!F.hasName()) {
        SmallString<64> Name;
        Mangler::getNameWithPrefix(Name, "__amdgpu_enqueued_kernel",
                                   M.getDataLayout());
        F.setName(Name);
      }
      LLVM_DEBUG(dbgs() << "found enqueued kernel: " << F.getName() << '\n');
      auto RuntimeHandle = (F.getName() + ".runtime_handle").str();
      if (!HandleTy) {
        Type *Int32 = Type::getInt32Ty(C);
        HandleTy =
            StructType::create(C, {PointerType::getUnqual(C), Int32, Int32},
                               "block.runtime.handle.t");
      }
 
      auto *GV = new GlobalVariable(
          M, HandleTy,
          /*isConstant=*/true, GlobalValue::ExternalLinkage,
          /*Initializer=*/Constant::getNullValue(HandleTy), RuntimeHandle,
          /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
          AMDGPUAS::GLOBAL_ADDRESS,
          /*isExternallyInitialized=*/true);
      LLVM_DEBUG(dbgs() << "runtime handle created: " << *GV << '\n');
 
      F.replaceAllUsesWith(ConstantExpr::getAddrSpaceCast(GV, F.getType()));
      F.addFnAttr("runtime-handle", RuntimeHandle);
      F.setLinkage(GlobalValue::ExternalLinkage);
      Changed = true;
    }
  }
 
  return Changed;
}