diff --git a/Tensile/Source/lib/include/Tensile/hip/HipSolutionAdapter.hpp b/Tensile/Source/lib/include/Tensile/hip/HipSolutionAdapter.hpp index 99850ef9e..25c5b2c47 100644 --- a/Tensile/Source/lib/include/Tensile/hip/HipSolutionAdapter.hpp +++ b/Tensile/Source/lib/include/Tensile/hip/HipSolutionAdapter.hpp @@ -87,6 +87,7 @@ namespace Tensile std::mutex m_access; std::vector m_modules; + std::vector> m_moduleBuffers; std::unordered_map m_kernels; bool m_debug = false; bool m_debugSkipLaunch = false; diff --git a/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp b/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp index dd7cf7d94..bc6b5449f 100644 --- a/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp +++ b/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -81,9 +82,27 @@ namespace Tensile hipError_t SolutionAdapter::loadCodeObjectFile(std::string const& path) { - hipModule_t module; + hipModule_t module; + std::unique_ptr buffer; + std::ifstream coFile(path, std::ifstream::binary); + + // hipModuleLoad holds the file descriptor/handle which can result in a process + // running out of descriptors/handles. Use hipModuleLoadData as a workaround + if(coFile) + { + coFile.seekg(0, coFile.end); + auto length = coFile.tellg(); + coFile.seekg(0, coFile.beg); - HIP_CHECK_RETURN(hipModuleLoad(&module, path.c_str())); + buffer = std::make_unique(length); + coFile.read(buffer.get(), length); + + HIP_CHECK_RETURN(hipModuleLoadData(&module, (void*)buffer.get())); + } + else + { + return hipErrorFileNotFound; + } if(m_debug) std::cout << "loaded code object " << path << std::endl; @@ -93,6 +112,9 @@ namespace Tensile m_modules.push_back(module); m_loadedModuleNames.push_back(concatenate("File ", path)); + // hipModuleLoadData requires the buffer to outlive the module, so cache the buffer + m_moduleBuffers.push_back(std::move(buffer)); + //Isolate filename size_t start = path.rfind('/'); start = (start == std::string::npos) ? 0 : start + 1; @@ -242,7 +264,7 @@ namespace Tensile for(auto ver : {"", "-xnack-", "-xnack+"}) { std::string modifiedCOName = helperKernelName + ver + ".hsaco"; - err = loadCodeObjectFile(codeObjectDir + modifiedCOName); + err = loadCodeObjectFile(codeObjectDir + modifiedCOName); if(err == hipSuccess) return err;