From 2ee449e124291d0ef69b89fa285e494cc087c9d5 Mon Sep 17 00:00:00 2001 From: Nicklas Lindgren Date: Sat, 23 Jul 2016 16:02:20 +0200 Subject: [PATCH] Add vertex shader dumping/override This change generalizes the pixel shader dumping and override support to also include vertex shaders. --- DATA/DSfix.ini | 15 ++++---- RenderstateManager.cpp | 82 ++++++++++++++++++++++++++++++------------ RenderstateManager.h | 7 ++-- Settings.def | 6 ++-- d3d9dev.cpp | 9 +---- main.cpp | 10 +++++- main.h | 3 +- 7 files changed, 87 insertions(+), 45 deletions(-) diff --git a/DATA/DSfix.ini b/DATA/DSfix.ini index 06f8806..5d1bd8d 100644 --- a/DATA/DSfix.ini +++ b/DATA/DSfix.ini @@ -201,14 +201,17 @@ enableTexturePrefetch 0 # Shader Override Options ############################################################################### -# Enable pixel shader dumping -# Disassembled pixel shaders will be dumped to "dsfix\pixelshader_dump\[hash].asm". -enablePixelShaderDumping 0 +# Enable shader dumping +# Disassembled pixel shaders will be dumped to +# "dsfix\pixelshader_dump\[hash].asm", vertex shaders to +# "dsfix\vertexshader_dump\[hash].asm". +enableShaderDumping 0 # Enable pixel shader override -# Pixel shaders named "dsfix\pixelshader_override\[hash].asm" will -# replace the corresponding originals. -enablePixelShaderOverride 0 +# Shaders from files named "dsfix\pixelshader_override\[hash].asm" or +# "dsfix\vertexshader_override\[hash].asm" will replace the +# corresponding originals. +enableShaderOverride 0 ############################################################################### # Other Options diff --git a/RenderstateManager.cpp b/RenderstateManager.cpp index fd6b170..5d21ccc 100644 --- a/RenderstateManager.cpp +++ b/RenderstateManager.cpp @@ -15,8 +15,10 @@ RSManager RSManager::instance; -#define PIXEL_SHADER_DUMP_DIR "dsfix/pixelshader_dump" -#define PIXEL_SHADER_OVERRIDE_DIR "dsfix/pixelshader_override" +static const char *PIXEL_SHADER_DUMP_DIR = "dsfix/pixelshader_dump"; +static const char *PIXEL_SHADER_OVERRIDE_DIR = "dsfix/pixelshader_override"; +static const char *VERTEX_SHADER_DUMP_DIR = "dsfix/vertexshader_dump"; +static const char *VERTEX_SHADER_OVERRIDE_DIR = "dsfix/vertexshader_override"; void RSManager::initResources() { SDLOG(0, "RenderstateManager resource initialization started\n"); @@ -40,12 +42,9 @@ void RSManager::initResources() { if (Settings::get().getEnableTextureOverride() && Settings::get().getEnableTexturePrefetch()) prefetchTextures(); - if (Settings::get().getEnablePixelShaderDumping()) { - CreateDirectory(GetDirectoryFile(PIXEL_SHADER_DUMP_DIR), nullptr); - DWORD error = GetLastError(); - if (error && error != ERROR_ALREADY_EXISTS) { - SDLOG(0, "Failed to create %s: %s\n", PIXEL_SHADER_DUMP_DIR, formatMessage(error)); - } + if (Settings::get().getEnableShaderDumping()) { + createDirectory(PIXEL_SHADER_DUMP_DIR); + createDirectory(VERTEX_SHADER_DUMP_DIR); } SDLOG(0, "RenderstateManager resource initialization completed\n"); @@ -869,14 +868,14 @@ HRESULT RSManager::redirectSetRenderState(D3DRENDERSTATETYPE State, DWORD Value) HRESULT RSManager::redirectCreatePixelShader(CONST DWORD* pFunction, IDirect3DPixelShader9** ppShader) { - bool shouldDump = Settings::get().getEnablePixelShaderDumping(); - bool shouldOverride = Settings::get().getEnablePixelShaderOverride(); + bool shouldDump = Settings::get().getEnableShaderDumping(); + bool shouldOverride = Settings::get().getEnableShaderOverride(); LPD3DXBUFFER pAssemblerBuffer = nullptr; LPD3DXBUFFER pFunctionBuffer = nullptr; UINT32 hash; if (shouldDump || shouldOverride) { - if (disassemblePixelShader(pFunction, &pAssemblerBuffer)) { + if (disassembleShader(pFunction, &pAssemblerBuffer)) { hash = SuperFastHash(static_cast(pAssemblerBuffer->GetBufferPointer()), pAssemblerBuffer->GetBufferSize()); } else { @@ -886,11 +885,11 @@ HRESULT RSManager::redirectCreatePixelShader(CONST DWORD* pFunction, IDirect3DPi } if (shouldDump) { - dumpPixelShader(hash, pAssemblerBuffer); + dumpShader(hash, PIXEL_SHADER_DUMP_DIR, pAssemblerBuffer); } if (shouldOverride) { - if (getOverridePixelShader(hash, &pFunctionBuffer)) { + if (getOverrideShader(hash, PIXEL_SHADER_OVERRIDE_DIR, &pFunctionBuffer)) { pFunction = static_cast(pFunctionBuffer->GetBufferPointer()); } } @@ -904,33 +903,70 @@ HRESULT RSManager::redirectCreatePixelShader(CONST DWORD* pFunction, IDirect3DPi return result; } -bool RSManager::disassemblePixelShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer) { +HRESULT RSManager::redirectCreateVertexShader(CONST DWORD* pFunction, IDirect3DVertexShader9** ppShader) +{ + bool shouldDump = Settings::get().getEnableShaderDumping(); + bool shouldOverride = Settings::get().getEnableShaderOverride(); + LPD3DXBUFFER pAssemblerBuffer = nullptr; + LPD3DXBUFFER pFunctionBuffer = nullptr; + UINT32 hash; + + if (shouldDump || shouldOverride) { + if (disassembleShader(pFunction, &pAssemblerBuffer)) { + hash = SuperFastHash(static_cast(pAssemblerBuffer->GetBufferPointer()), pAssemblerBuffer->GetBufferSize()); + } + else { + shouldDump = false; + shouldOverride = false; + } + } + + if (shouldDump) { + dumpShader(hash, VERTEX_SHADER_DUMP_DIR, pAssemblerBuffer); + } + + if (shouldOverride) { + if (getOverrideShader(hash, VERTEX_SHADER_OVERRIDE_DIR, &pFunctionBuffer)) { + pFunction = static_cast(pFunctionBuffer->GetBufferPointer()); + } + } + + HRESULT result = d3ddev->CreateVertexShader(pFunction, ppShader); + if (shouldDump || shouldOverride) { + SDLOG(1, "Created vertex shader for hash %08x: 0x%p\n", hash, *ppShader); + } + SAFERELEASE(pAssemblerBuffer); + SAFERELEASE(pFunctionBuffer); + return result; +} + +bool RSManager::disassembleShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer) { LPD3DXBUFFER buffer = nullptr; HRESULT result = D3DXDisassembleShader(pFunction, false, NULL, ppBuffer); if (result != D3D_OK) { - SDLOG(0, "Failed to disassemble pixel shader\n"); + SDLOG(0, "Failed to disassemble shader\n"); } return result == D3D_OK; } -void RSManager::dumpPixelShader(UINT32 hash, LPD3DXBUFFER pBuffer) { +void RSManager::dumpShader(UINT32 hash, const char *directory, LPD3DXBUFFER pBuffer) { char fileNameBuffer[64]; - sprintf_s(fileNameBuffer, PIXEL_SHADER_DUMP_DIR "/%08x.asm", hash); + sprintf_s(fileNameBuffer, "%s/%08x.asm", directory, hash); const char *fileName = GetDirectoryFile(fileNameBuffer); - if (writeFile(fileName, static_cast(pBuffer->GetBufferPointer()), pBuffer->GetBufferSize())) { - SDLOG(0, "Wrote disassembled pixel shader to %s\n", fileNameBuffer); + if (writeFile(fileName, static_cast(pBuffer->GetBufferPointer()), pBuffer->GetBufferSize() - 1)) { + SDLOG(0, "Wrote disassembled shader to %s\n", fileNameBuffer); } else { - SDLOG(0, "Failed to write disassembled pixel shader to %s\n", fileNameBuffer); + SDLOG(0, "Failed to write disassembled shader to %s\n", fileNameBuffer); } } -bool RSManager::getOverridePixelShader(UINT32 hash, LPD3DXBUFFER *ppBuffer) { +bool RSManager::getOverrideShader(UINT32 hash, const char *directory, LPD3DXBUFFER *ppBuffer) { char fileNameBuffer[64]; - sprintf_s(fileNameBuffer, PIXEL_SHADER_OVERRIDE_DIR "/%08x.asm", hash); + sprintf_s(fileNameBuffer, "%s/%08x.asm", directory, hash); const char * fileName = GetDirectoryFile(fileNameBuffer); if (fileExists(fileName)) { - SDLOG(1, "Pixel shader override: %08x\n", hash); + SDLOG(1, "Shader override: %s\n", fileName); LPD3DXBUFFER errors = nullptr; HRESULT assembleResult = D3DXAssembleShaderFromFile(fileName, nullptr, nullptr, 0, ppBuffer, &errors); if (assembleResult != D3D_OK) { diff --git a/RenderstateManager.h b/RenderstateManager.h index 5fddbb5..63c950c 100644 --- a/RenderstateManager.h +++ b/RenderstateManager.h @@ -113,9 +113,9 @@ class RSManager { std::map cachedTexFiles; - bool disassemblePixelShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer); - void dumpPixelShader(UINT32 hash, LPD3DXBUFFER pBuffer); - bool getOverridePixelShader(UINT32 hash, LPD3DXBUFFER *ppBuffer); + bool disassembleShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer); + void dumpShader(UINT32 hash, const char *directory, LPD3DXBUFFER pBuffer); + bool getOverrideShader(UINT32 hash, const char *directory, LPD3DXBUFFER *ppBuffer); private: ~RSManager(); @@ -194,4 +194,5 @@ class RSManager { HRESULT redirectSetTextureStageState(DWORD Stage, D3DTEXTURESTAGESTATETYPE Type, DWORD Value); HRESULT redirectSetRenderState(D3DRENDERSTATETYPE State, DWORD Value); HRESULT redirectCreatePixelShader(CONST DWORD *pfunction, IDirect3DPixelShader9 **ppShader); + HRESULT redirectCreateVertexShader(CONST DWORD *pfunction, IDirect3DVertexShader9 **ppShader); }; diff --git a/Settings.def b/Settings.def index 573466f..ff0b35b 100644 --- a/Settings.def +++ b/Settings.def @@ -52,9 +52,9 @@ SETTING(bool, EnableTextureDumping, "enableTextureDumping", false); SETTING(bool, EnableTextureOverride, "enableTextureOverride", false); SETTING(bool, EnableTexturePrefetch, "enableTexturePrefetch", false); -// Pixel Shader Override Options -SETTING(bool, EnablePixelShaderDumping, "enablePixelShaderDumping", false); -SETTING(bool, EnablePixelShaderOverride, "enablePixelShaderOverride", false); +// Shader Override Options +SETTING(bool, EnableShaderDumping, "enableShaderDumping", false); +SETTING(bool, EnableShaderOverride, "enableShaderOverride", false); // HUD options SETTING(bool, EnableHudMod, "enableHudMod", false) diff --git a/d3d9dev.cpp b/d3d9dev.cpp index df784ec..dd8da0e 100644 --- a/d3d9dev.cpp +++ b/d3d9dev.cpp @@ -253,14 +253,7 @@ HRESULT APIENTRY hkIDirect3DDevice9::CreateVertexDeclaration(CONST D3DVERTEXELEM } HRESULT APIENTRY hkIDirect3DDevice9::CreateVertexShader(CONST DWORD* pFunction,IDirect3DVertexShader9** ppShader) { - HRESULT res = m_pD3Ddev->CreateVertexShader(pFunction, ppShader); - if(Settings::get().getLogLevel() > 12) { - SDLOG(12, "CreateVertexShader data: %p : shader: %p\n", pFunction, *ppShader); - LPD3DXBUFFER buffer; - D3DXDisassembleShader(pFunction, false, NULL, &buffer); - SDLOG(12, "===== disassembly:\n%s\n==============\n", buffer->GetBufferPointer()); - } - return res; + return RSManager::get().redirectCreateVertexShader(pFunction, ppShader); } HRESULT APIENTRY hkIDirect3DDevice9::CreateVolumeTexture(UINT Width,UINT Height,UINT Depth,UINT Levels,DWORD Usage,D3DFORMAT Format,D3DPOOL Pool,IDirect3DVolumeTexture9** ppVolumeTexture,HANDLE* pSharedHandle) { diff --git a/main.cpp b/main.cpp index 7b6661c..c8c62cb 100644 --- a/main.cpp +++ b/main.cpp @@ -79,7 +79,7 @@ bool WINAPI DllMain(HMODULE hDll, DWORD dwReason, PVOID pvReserved) { return false; } -char *GetDirectoryFile(char *filename) { +char *GetDirectoryFile(const char *filename) { static char path[320]; strcpy_s(path, dlldir); strcat_s(path, filename); @@ -136,6 +136,14 @@ bool fileExists(const char *filename) { return std::ifstream(filename).good(); } +void createDirectory(const char *fileName) { + CreateDirectory(GetDirectoryFile(fileName), nullptr); + DWORD error = GetLastError(); + if (error && error != ERROR_ALREADY_EXISTS) { + SDLOG(0, "Failed to create %s: %s\n", fileName, formatMessage(error)); + } +} + bool writeFile(const char *filename, const char *data, size_t length) { std::ofstream file(filename, std::ios::out | std::ios::binary); if (!file) { diff --git a/main.h b/main.h index a748410..0b02c30 100644 --- a/main.h +++ b/main.h @@ -36,8 +36,9 @@ #include "dinput.h" #include -char *GetDirectoryFile(char *filename); +char *GetDirectoryFile(const char *filename); bool fileExists(const char *filename); +void createDirectory(const char *filename); bool writeFile(const char *filename, const char *data, size_t length); std::string formatMessage(DWORD messageId); std::string strError(int err);