Merge pull request PeterTh#31 from niligulmohar/dump-or-override-shaders

Add shader dumping/override

DATA/DSfix.ini

@@ -197,6 +197,22 @@ enableTextureOverride 0
 # no disk overhead when loading them when entering new area (little performance boost)
 enableTexturePrefetch 0
 
+###############################################################################
+# Shader Override Options
+###############################################################################
+
+# Enable shader dumping
+# Disassembled pixel shaders will be dumped to
+# "dsfix\pixelshader_dump\[hash].asm", vertex shaders to
+# "dsfix\vertexshader_dump\[hash].asm".
+enableShaderDumping 0
+
+# Enable pixel shader override
+# Shaders from files named "dsfix\pixelshader_override\[hash].asm" or
+# "dsfix\vertexshader_override\[hash].asm" will replace the
+# corresponding originals.
+enableShaderOverride 0
+
 ###############################################################################
 # Other Options
 ###############################################################################

RenderstateManager.cpp

@@ -15,6 +15,11 @@
 
 RSManager RSManager::instance;
 
+static const char *PIXEL_SHADER_DUMP_DIR = "dsfix/pixelshader_dump";
+static const char *PIXEL_SHADER_OVERRIDE_DIR = "dsfix/pixelshader_override";
+static const char *VERTEX_SHADER_DUMP_DIR = "dsfix/vertexshader_dump";
+static const char *VERTEX_SHADER_OVERRIDE_DIR = "dsfix/vertexshader_override";
+
 void RSManager::initResources() {
 	SDLOG(0, "RenderstateManager resource initialization started\n");
 	unsigned rw = Settings::get().getRenderWidth(), rh = Settings::get().getRenderHeight();
@@ -36,6 +41,12 @@ void RSManager::initResources() {
 	d3ddev->CreateStateBlock(D3DSBT_ALL, &prevStateBlock);
     if (Settings::get().getEnableTextureOverride() && Settings::get().getEnableTexturePrefetch())
         prefetchTextures();
+
+	if (Settings::get().getEnableShaderDumping()) {
+		createDirectory(PIXEL_SHADER_DUMP_DIR);
+		createDirectory(VERTEX_SHADER_DUMP_DIR);
+	}
+
 	SDLOG(0, "RenderstateManager resource initialization completed\n");
 	if(!inited) startDetour(); // on first init only
 	inited = true;
@@ -855,6 +866,118 @@ HRESULT RSManager::redirectSetRenderState(D3DRENDERSTATETYPE State, DWORD Value)
 	//return D3D_OK;
 }
 
+HRESULT RSManager::redirectCreatePixelShader(CONST DWORD* pFunction, IDirect3DPixelShader9** ppShader)
+{
+	bool shouldDump = Settings::get().getEnableShaderDumping();
+	bool shouldOverride = Settings::get().getEnableShaderOverride();
+	LPD3DXBUFFER pAssemblerBuffer = nullptr;
+	LPD3DXBUFFER pFunctionBuffer = nullptr;
+	UINT32 hash;
+
+	if (shouldDump || shouldOverride) {
+		if (disassembleShader(pFunction, &pAssemblerBuffer)) {
+			hash = SuperFastHash(static_cast<char *>(pAssemblerBuffer->GetBufferPointer()), pAssemblerBuffer->GetBufferSize());
+		}
+		else {
+			shouldDump = false;
+			shouldOverride = false;
+		}
+	}
+
+	if (shouldDump) {
+		dumpShader(hash, PIXEL_SHADER_DUMP_DIR, pAssemblerBuffer);
+	}
+
+	if (shouldOverride) {
+		if (getOverrideShader(hash, PIXEL_SHADER_OVERRIDE_DIR, &pFunctionBuffer)) {
+			pFunction = static_cast<DWORD *>(pFunctionBuffer->GetBufferPointer());
+		}
+	}
+
+	HRESULT result = d3ddev->CreatePixelShader(pFunction, ppShader);
+	if (shouldDump || shouldOverride) {
+		SDLOG(1, "Created pixel shader for hash %08x: 0x%p\n", hash, *ppShader);
+	}
+	SAFERELEASE(pAssemblerBuffer);
+	SAFERELEASE(pFunctionBuffer);
+	return result;
+}
+
+HRESULT RSManager::redirectCreateVertexShader(CONST DWORD* pFunction, IDirect3DVertexShader9** ppShader)
+{
+	bool shouldDump = Settings::get().getEnableShaderDumping();
+	bool shouldOverride = Settings::get().getEnableShaderOverride();
+	LPD3DXBUFFER pAssemblerBuffer = nullptr;
+	LPD3DXBUFFER pFunctionBuffer = nullptr;
+	UINT32 hash;
+
+	if (shouldDump || shouldOverride) {
+		if (disassembleShader(pFunction, &pAssemblerBuffer)) {
+			hash = SuperFastHash(static_cast<char *>(pAssemblerBuffer->GetBufferPointer()), pAssemblerBuffer->GetBufferSize());
+		}
+		else {
+			shouldDump = false;
+			shouldOverride = false;
+		}
+	}
+
+	if (shouldDump) {
+		dumpShader(hash, VERTEX_SHADER_DUMP_DIR, pAssemblerBuffer);
+	}
+
+	if (shouldOverride) {
+		if (getOverrideShader(hash, VERTEX_SHADER_OVERRIDE_DIR, &pFunctionBuffer)) {
+			pFunction = static_cast<DWORD *>(pFunctionBuffer->GetBufferPointer());
+		}
+	}
+
+	HRESULT result = d3ddev->CreateVertexShader(pFunction, ppShader);
+	if (shouldDump || shouldOverride) {
+		SDLOG(1, "Created vertex shader for hash %08x: 0x%p\n", hash, *ppShader);
+	}
+	SAFERELEASE(pAssemblerBuffer);
+	SAFERELEASE(pFunctionBuffer);
+	return result;
+}
+
+bool RSManager::disassembleShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer) {
+	LPD3DXBUFFER buffer = nullptr;
+	HRESULT result = D3DXDisassembleShader(pFunction, false, NULL, ppBuffer);
+	if (result != D3D_OK) {
+		SDLOG(0, "Failed to disassemble shader\n");
+	}
+	return result == D3D_OK;
+}
+
+void RSManager::dumpShader(UINT32 hash, const char *directory, LPD3DXBUFFER pBuffer) {
+	char fileNameBuffer[64];
+	sprintf_s(fileNameBuffer, "%s/%08x.asm", directory, hash);
+	const char *fileName = GetDirectoryFile(fileNameBuffer);
+	if (writeFile(fileName, static_cast<char *>(pBuffer->GetBufferPointer()), pBuffer->GetBufferSize() - 1)) {
+		SDLOG(0, "Wrote disassembled shader to %s\n", fileNameBuffer);
+	}
+	else {
+		SDLOG(0, "Failed to write disassembled shader to %s\n", fileNameBuffer);
+	}
+}
+
+bool RSManager::getOverrideShader(UINT32 hash, const char *directory, LPD3DXBUFFER *ppBuffer) {
+	char fileNameBuffer[64];
+	sprintf_s(fileNameBuffer, "%s/%08x.asm", directory, hash);
+	const char * fileName = GetDirectoryFile(fileNameBuffer);
+	if (fileExists(fileName)) {
+		SDLOG(1, "Shader override: %s\n", fileName);
+		LPD3DXBUFFER errors = nullptr;
+		HRESULT assembleResult = D3DXAssembleShaderFromFile(fileName, nullptr, nullptr, 0, ppBuffer, &errors);
+		if (assembleResult != D3D_OK) {
+			SDLOG(0, "Failed to assemble replacement shader:\n%s\n", errors->GetBufferPointer());
+		}
+		SAFERELEASE(errors);
+		return assembleResult == D3D_OK;
+	}
+	return false;
+}
+
 void RSManager::frameTimeManagement() {
 	double renderTime = getElapsedTime() - lastPresentTime;
 

RenderstateManager.h

@@ -112,6 +112,10 @@ class RSManager {
     };
 
     std::map<UINT32, MemData> cachedTexFiles;
+
+	bool disassembleShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer);
+	void dumpShader(UINT32 hash, const char *directory, LPD3DXBUFFER pBuffer);
+	bool getOverrideShader(UINT32 hash, const char *directory, LPD3DXBUFFER *ppBuffer);
 
 private:
     ~RSManager();
@@ -189,4 +193,6 @@ class RSManager {
 	HRESULT redirectD3DXCreateTextureFromFileInMemoryEx(LPDIRECT3DDEVICE9 pDevice, LPCVOID pSrcData, UINT SrcDataSize, UINT Width, UINT Height, UINT MipLevels, DWORD Usage, D3DFORMAT Format, D3DPOOL Pool, DWORD Filter, DWORD MipFilter, D3DCOLOR ColorKey, D3DXIMAGE_INFO* pSrcInfo, PALETTEENTRY* pPalette, LPDIRECT3DTEXTURE9* ppTexture);
 	HRESULT redirectSetTextureStageState(DWORD Stage, D3DTEXTURESTAGESTATETYPE Type, DWORD Value);
 	HRESULT redirectSetRenderState(D3DRENDERSTATETYPE State, DWORD Value);
+	HRESULT redirectCreatePixelShader(CONST DWORD *pfunction, IDirect3DPixelShader9 **ppShader);
+	HRESULT redirectCreateVertexShader(CONST DWORD *pfunction, IDirect3DVertexShader9 **ppShader);
 };

Settings.def

@@ -52,6 +52,10 @@ SETTING(bool, EnableTextureDumping, "enableTextureDumping", false);
 SETTING(bool, EnableTextureOverride, "enableTextureOverride", false);
 SETTING(bool, EnableTexturePrefetch, "enableTexturePrefetch", false);
 
+// Shader Override Options
+SETTING(bool, EnableShaderDumping, "enableShaderDumping", false);
+SETTING(bool, EnableShaderOverride, "enableShaderOverride", false);
+
 // HUD options
 SETTING(bool, EnableHudMod, "enableHudMod", false)
 SETTING(bool, EnableMinimalHud, "enableMinimalHud", false)

d3d9dev.cpp

@@ -182,14 +182,7 @@ HRESULT APIENTRY hkIDirect3DDevice9::CreateOffscreenPlainSurface(UINT Width, UIN
 }
 
 HRESULT APIENTRY hkIDirect3DDevice9::CreatePixelShader(CONST DWORD* pFunction,IDirect3DPixelShader9** ppShader) {
-	HRESULT res = m_pD3Ddev->CreatePixelShader(pFunction, ppShader);
-	if(Settings::get().getLogLevel() > 12) {
-		SDLOG(12, "CreatePixelShader data: %p : shader: %p\n", pFunction, *ppShader);
-		LPD3DXBUFFER buffer;
-		D3DXDisassembleShader(pFunction, false, NULL, &buffer);
-		SDLOG(12, "===== disassembly:\n%s\n==============\n", buffer->GetBufferPointer());
-	}
-	return res;
+	return RSManager::get().redirectCreatePixelShader(pFunction, ppShader);
 }
 
 HRESULT APIENTRY hkIDirect3DDevice9::CreateQuery(D3DQUERYTYPE Type,IDirect3DQuery9** ppQuery) {
@@ -260,14 +253,7 @@ HRESULT APIENTRY hkIDirect3DDevice9::CreateVertexDeclaration(CONST D3DVERTEXELEM
 }
 
 HRESULT APIENTRY hkIDirect3DDevice9::CreateVertexShader(CONST DWORD* pFunction,IDirect3DVertexShader9** ppShader) {
-	HRESULT res = m_pD3Ddev->CreateVertexShader(pFunction, ppShader);
-	if(Settings::get().getLogLevel() > 12) {
-		SDLOG(12, "CreateVertexShader data: %p : shader: %p\n", pFunction, *ppShader);
-		LPD3DXBUFFER buffer;
-		D3DXDisassembleShader(pFunction, false, NULL, &buffer);
-		SDLOG(12, "===== disassembly:\n%s\n==============\n", buffer->GetBufferPointer());
-	}
-	return res;
+	return RSManager::get().redirectCreateVertexShader(pFunction, ppShader);
 }
 
 HRESULT APIENTRY hkIDirect3DDevice9::CreateVolumeTexture(UINT Width,UINT Height,UINT Depth,UINT Levels,DWORD Usage,D3DFORMAT Format,D3DPOOL Pool,IDirect3DVolumeTexture9** ppVolumeTexture,HANDLE* pSharedHandle) {

main.cpp

@@ -80,7 +80,7 @@ bool WINAPI DllMain(HMODULE hDll, DWORD dwReason, PVOID pvReserved) {
     return false;
 }
 
-char *GetDirectoryFile(char *filename) {
+char *GetDirectoryFile(const char *filename) {
 	static char path[320];
 	strcpy_s(path, dlldir);
 	strcat_s(path, filename);
@@ -136,3 +136,45 @@ void errorExit(LPTSTR lpszFunction) {
 bool fileExists(const char *filename) {
   return std::ifstream(filename).good();
 }
+
+void createDirectory(const char *fileName) {
+	CreateDirectory(GetDirectoryFile(fileName), nullptr);
+	DWORD error = GetLastError();
+	if (error && error != ERROR_ALREADY_EXISTS) {
+		SDLOG(0, "Failed to create %s: %s\n", fileName, formatMessage(error));
+	}
+}
+
+bool writeFile(const char *filename, const char *data, size_t length) {
+	std::ofstream file(filename, std::ios::out | std::ios::binary);
+	if (!file) {
+		SDLOG(0, "Failed to open %s: %s\n", filename, strError(errno));
+		return false;
+	}
+	file.write(data, length);
+	if (!file) {
+		SDLOG(0, "Failed to write to %s: %s\n", filename, strError(errno));
+		return false;
+	}
+	file.close();
+	if (!file) {
+		SDLOG(0, "Failed to close %s: %s\n", filename, strError(errno));
+		return false;
+	}
+	return true;
+}
+
+std::string formatMessage(DWORD messageId) {
+	char *buffer = nullptr;
+	size_t length = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+		nullptr, messageId, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&buffer, 0, nullptr);
+	std::string result(buffer, length);
+	LocalFree(buffer);
+	return result;
+}
+
+std::string strError(int err) {
+	std::string result(4096, '\0');
+	strerror_s(&result[0], result.length(), err);
+	return result;
+}

main.h

@@ -34,9 +34,14 @@
 
 #include "d3d9.h"
 #include "dinput.h"
+#include <string>
 
-char *GetDirectoryFile(char *filename);
+char *GetDirectoryFile(const char *filename);
 bool fileExists(const char *filename);
+void createDirectory(const char *filename);
+bool writeFile(const char *filename, const char *data, size_t length);
+std::string formatMessage(DWORD messageId);
+std::string strError(int err);
 void __cdecl sdlogtime();
 void __cdecl sdlog(const char * fmt, ...);
 void errorExit(LPTSTR lpszFunction);