Add pixel shader dumping/override

This change adds configurable dumping and overriding of pixel shaders as
assembler, similar to the texture dumping and overriding.

DATA/DSfix.ini

@@ -197,6 +197,19 @@ enableTextureOverride 0
 # no disk overhead when loading them when entering new area (little performance boost)
 enableTexturePrefetch 0
 
+###############################################################################
+# Shader Override Options
+###############################################################################
+
+# Enable pixel shader dumping
+# Disassembled pixel shaders will be dumped to "dsfix\pixelshader_dump\[hash].asm".
+enablePixelShaderDumping 0
+
+# Enable pixel shader override
+# Pixel shaders named "dsfix\pixelshader_override\[hash].asm" will
+# replace the corresponding originals.
+enablePixelShaderOverride 0
+
 ###############################################################################
 # Other Options
 ###############################################################################

RenderstateManager.cpp

@@ -15,6 +15,9 @@
 
 RSManager RSManager::instance;
 
+#define PIXEL_SHADER_DUMP_DIR "dsfix/pixelshader_dump"
+#define PIXEL_SHADER_OVERRIDE_DIR "dsfix/pixelshader_override"
+
 void RSManager::initResources() {
 	SDLOG(0, "RenderstateManager resource initialization started\n");
 	unsigned rw = Settings::get().getRenderWidth(), rh = Settings::get().getRenderHeight();
@@ -36,6 +39,15 @@ void RSManager::initResources() {
 	d3ddev->CreateStateBlock(D3DSBT_ALL, &prevStateBlock);
     if (Settings::get().getEnableTextureOverride() && Settings::get().getEnableTexturePrefetch())
         prefetchTextures();
+
+	if (Settings::get().getEnablePixelShaderDumping()) {
+		CreateDirectory(GetDirectoryFile(PIXEL_SHADER_DUMP_DIR), nullptr);
+		DWORD error = GetLastError();
+		if (error && error != ERROR_ALREADY_EXISTS) {
+			SDLOG(0, "Failed to create %s: %s\n", PIXEL_SHADER_DUMP_DIR, formatMessage(error));
+		}
+	}
+
 	SDLOG(0, "RenderstateManager resource initialization completed\n");
 	if(!inited) startDetour(); // on first init only
 	inited = true;
@@ -855,6 +867,81 @@ HRESULT RSManager::redirectSetRenderState(D3DRENDERSTATETYPE State, DWORD Value)
 	//return D3D_OK;
 }
 
+HRESULT RSManager::redirectCreatePixelShader(CONST DWORD* pFunction, IDirect3DPixelShader9** ppShader)
+{
+	bool shouldDump = Settings::get().getEnablePixelShaderDumping();
+	bool shouldOverride = Settings::get().getEnablePixelShaderOverride();
+	LPD3DXBUFFER pAssemblerBuffer = nullptr;
+	LPD3DXBUFFER pFunctionBuffer = nullptr;
+	UINT32 hash;
+
+	if (shouldDump || shouldOverride) {
+		if (disassemblePixelShader(pFunction, &pAssemblerBuffer)) {
+			hash = SuperFastHash(static_cast<char *>(pAssemblerBuffer->GetBufferPointer()), pAssemblerBuffer->GetBufferSize());
+		}
+		else {
+			shouldDump = false;
+			shouldOverride = false;
+		}
+	}
+
+	if (shouldDump) {
+		dumpPixelShader(hash, pAssemblerBuffer);
+	}
+
+	if (shouldOverride) {
+		if (getOverridePixelShader(hash, &pFunctionBuffer)) {
+			pFunction = static_cast<DWORD *>(pFunctionBuffer->GetBufferPointer());
+		}
+	}
+
+	HRESULT result = d3ddev->CreatePixelShader(pFunction, ppShader);
+	if (shouldDump || shouldOverride) {
+		SDLOG(1, "Created pixel shader for hash %08x: 0x%p\n", hash, *ppShader);
+	}
+	SAFERELEASE(pAssemblerBuffer);
+	SAFERELEASE(pFunctionBuffer);
+	return result;
+}
+
+bool RSManager::disassemblePixelShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer) {
+	LPD3DXBUFFER buffer = nullptr;
+	HRESULT result = D3DXDisassembleShader(pFunction, false, NULL, ppBuffer);
+	if (result != D3D_OK) {
+		SDLOG(0, "Failed to disassemble pixel shader\n");
+	}
+	return result == D3D_OK;
+}
+
+void RSManager::dumpPixelShader(UINT32 hash, LPD3DXBUFFER pBuffer) {
+	char fileNameBuffer[64];
+	sprintf_s(fileNameBuffer, PIXEL_SHADER_DUMP_DIR "/%08x.asm", hash);
+	const char *fileName = GetDirectoryFile(fileNameBuffer);
+	if (writeFile(fileName, static_cast<char *>(pBuffer->GetBufferPointer()), pBuffer->GetBufferSize())) {
+		SDLOG(0, "Wrote disassembled pixel shader to %s\n", fileNameBuffer);
+	}
+	else {
+		SDLOG(0, "Failed to write disassembled pixel shader to %s\n", fileNameBuffer);
+	}
+}
+
+bool RSManager::getOverridePixelShader(UINT32 hash, LPD3DXBUFFER *ppBuffer) {
+	char fileNameBuffer[64];
+	sprintf_s(fileNameBuffer, PIXEL_SHADER_OVERRIDE_DIR "/%08x.asm", hash);
+	const char * fileName = GetDirectoryFile(fileNameBuffer);
+	if (fileExists(fileName)) {
+		SDLOG(1, "Pixel shader override: %08x\n", hash);
+		LPD3DXBUFFER errors = nullptr;
+		HRESULT assembleResult = D3DXAssembleShaderFromFile(fileName, nullptr, nullptr, 0, ppBuffer, &errors);
+		if (assembleResult != D3D_OK) {
+			SDLOG(0, "Failed to assemble replacement shader:\n%s\n", errors->GetBufferPointer());
+		}
+		SAFERELEASE(errors);
+		return assembleResult == D3D_OK;
+	}
+	return false;
+}
+
 void RSManager::frameTimeManagement() {
 	double renderTime = getElapsedTime() - lastPresentTime;
 

RenderstateManager.h

@@ -112,6 +112,10 @@ class RSManager {
     };
 
     std::map<UINT32, MemData> cachedTexFiles;
+
+	bool disassemblePixelShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer);
+	void dumpPixelShader(UINT32 hash, LPD3DXBUFFER pBuffer);
+	bool getOverridePixelShader(UINT32 hash, LPD3DXBUFFER *ppBuffer);
 
 private:
     ~RSManager();
@@ -189,4 +193,5 @@ class RSManager {
 	HRESULT redirectD3DXCreateTextureFromFileInMemoryEx(LPDIRECT3DDEVICE9 pDevice, LPCVOID pSrcData, UINT SrcDataSize, UINT Width, UINT Height, UINT MipLevels, DWORD Usage, D3DFORMAT Format, D3DPOOL Pool, DWORD Filter, DWORD MipFilter, D3DCOLOR ColorKey, D3DXIMAGE_INFO* pSrcInfo, PALETTEENTRY* pPalette, LPDIRECT3DTEXTURE9* ppTexture);
 	HRESULT redirectSetTextureStageState(DWORD Stage, D3DTEXTURESTAGESTATETYPE Type, DWORD Value);
 	HRESULT redirectSetRenderState(D3DRENDERSTATETYPE State, DWORD Value);
+	HRESULT redirectCreatePixelShader(CONST DWORD *pfunction, IDirect3DPixelShader9 **ppShader);
 };

Settings.def

@@ -52,6 +52,10 @@ SETTING(bool, EnableTextureDumping, "enableTextureDumping", false);
 SETTING(bool, EnableTextureOverride, "enableTextureOverride", false);
 SETTING(bool, EnableTexturePrefetch, "enableTexturePrefetch", false);
 
+// Pixel Shader Override Options
+SETTING(bool, EnablePixelShaderDumping, "enablePixelShaderDumping", false);
+SETTING(bool, EnablePixelShaderOverride, "enablePixelShaderOverride", false);
+
 // HUD options
 SETTING(bool, EnableHudMod, "enableHudMod", false)
 SETTING(bool, EnableMinimalHud, "enableMinimalHud", false)

d3d9dev.cpp

@@ -182,14 +182,7 @@ HRESULT APIENTRY hkIDirect3DDevice9::CreateOffscreenPlainSurface(UINT Width, UIN
 }
 
 HRESULT APIENTRY hkIDirect3DDevice9::CreatePixelShader(CONST DWORD* pFunction,IDirect3DPixelShader9** ppShader) {
-	HRESULT res = m_pD3Ddev->CreatePixelShader(pFunction, ppShader);
-	if(Settings::get().getLogLevel() > 12) {
-		SDLOG(12, "CreatePixelShader data: %p : shader: %p\n", pFunction, *ppShader);
-		LPD3DXBUFFER buffer;
-		D3DXDisassembleShader(pFunction, false, NULL, &buffer);
-		SDLOG(12, "===== disassembly:\n%s\n==============\n", buffer->GetBufferPointer());
-	}
-	return res;
+	return RSManager::get().redirectCreatePixelShader(pFunction, ppShader);
 }
 
 HRESULT APIENTRY hkIDirect3DDevice9::CreateQuery(D3DQUERYTYPE Type,IDirect3DQuery9** ppQuery) {

main.cpp

@@ -135,3 +135,37 @@ void errorExit(LPTSTR lpszFunction) {
 bool fileExists(const char *filename) {
   return std::ifstream(filename).good();
 }
+
+bool writeFile(const char *filename, const char *data, size_t length) {
+	std::ofstream file(filename, std::ios::out | std::ios::binary);
+	if (!file) {
+		SDLOG(0, "Failed to open %s: %s\n", filename, strError(errno));
+		return false;
+	}
+	file.write(data, length);
+	if (!file) {
+		SDLOG(0, "Failed to write to %s: %s\n", filename, strError(errno));
+		return false;
+	}
+	file.close();
+	if (!file) {
+		SDLOG(0, "Failed to close %s: %s\n", filename, strError(errno));
+		return false;
+	}
+	return true;
+}
+
+std::string formatMessage(DWORD messageId) {
+	char *buffer = nullptr;
+	size_t length = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+		nullptr, messageId, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&buffer, 0, nullptr);
+	std::string result(buffer, length);
+	LocalFree(buffer);
+	return result;
+}
+
+std::string strError(int err) {
+	std::string result(4096, '\0');
+	strerror_s(&result[0], result.length(), err);
+	return result;
+}

main.h

@@ -34,9 +34,13 @@
 
 #include "d3d9.h"
 #include "dinput.h"
+#include <string>
 
 char *GetDirectoryFile(char *filename);
 bool fileExists(const char *filename);
+bool writeFile(const char *filename, const char *data, size_t length);
+std::string formatMessage(DWORD messageId);
+std::string strError(int err);
 void __cdecl sdlogtime();
 void __cdecl sdlog(const char * fmt, ...);
 void errorExit(LPTSTR lpszFunction);