DX9: Hook up the depalettize code. Doesn't work yet though.

GPU/Common/DepalettizeShaderCommon.cpp

@@ -236,10 +236,10 @@ void GenerateDepalShaderFloat(char *buffer, GEBufferFormat pixelFormat, ShaderLa
 	} else if (lang == HLSL_DX9) {
 		WRITE(p, "sampler tex: register(s0);\n");
 		WRITE(p, "sampler pal: register(s1);\n");
-		WRITE(p, "float4 main(float2 v_texcoord0 : TEXCOORD0) {\n");
+		WRITE(p, "float4 main(float2 v_texcoord0 : TEXCOORD0) : COLOR0 {\n");
 		WRITE(p, "  float4 index = tex2D(tex, v_texcoord0);\n");
 		WRITE(p, "  float coord = (%s * %f)%s;\n", lookupMethod, index_multiplier, offset);
-		WRITE(p, "  gl_FragColor = tex2D(pal, float2(coord, 0.0));\n");
+		WRITE(p, "  return tex2D(pal, float2(coord, 0.0));\n");
 		WRITE(p, "}\n");
 	}
 }

GPU/Directx9/DepalettizeShaderDX9.cpp

@@ -40,13 +40,14 @@ static const char *depalVShaderHLSL =
 "  float2 a_texcoord0 : TEXCOORD0;\n"
 "};\n"
 "struct VS_OUT {\n"
-"  float3 Position : POSITION;\n"
+"  float4 Position : POSITION;\n"
 "  float2 Texcoord : TEXCOORD0;\n"
-"}\n"
-"VS_OUT main(VS_IN in) {\n"
-"  VS_OUT out;\n"
-"  out.Texcoord = in.a_texcoord0;\n"
-"  out.Position = in.a_position;\n"
+"};\n"
+"VS_OUT main(VS_IN input) {\n"
+"  VS_OUT output;\n"
+"  output.Texcoord = input.a_texcoord0;\n"
+"  output.Position = float4(input.a_position, 1.0);\n"
+"  return output;\n"
 "}\n";
 
 DepalShaderCacheDX9::DepalShaderCacheDX9() : vertexShader_(nullptr) {

GPU/Directx9/DepalettizeShaderDX9.h

@@ -43,7 +43,7 @@ public:
 
 	// This also uploads the palette and binds the correct texture.
 	LPDIRECT3DPIXELSHADER9 GetDepalettizePixelShader(GEBufferFormat pixelFormat);
-	LPDIRECT3DVERTEXSHADER9 GetDepalettizeVertexShader();
+	LPDIRECT3DVERTEXSHADER9 GetDepalettizeVertexShader() { return vertexShader_; }
 	LPDIRECT3DTEXTURE9 GetClutTexture(const u32 clutHash, u32 *rawClut);
 	void Clear();
 	void Decimate();

GPU/Directx9/GPU_DX9.cpp

@@ -403,6 +403,7 @@ DIRECTX9_GPU::DIRECTX9_GPU()
 	framebufferManager_.SetShaderManager(shaderManager_);
 	framebufferManager_.SetTransformDrawEngine(&transformDraw_);
 	textureCache_.SetFramebufferManager(&framebufferManager_);
+	textureCache_.SetDepalShaderCache(&depalShaderCache_);
 	textureCache_.SetShaderManager(shaderManager_);
 
 	// Sanity check gstate
@@ -525,7 +526,7 @@ void DIRECTX9_GPU::BeginFrameInternal() {
 
 	textureCache_.StartFrame();
 	transformDraw_.DecimateTrackedVertexArrays();
-	// depalShaderCache_.Decimate();
+	depalShaderCache_.Decimate();
 	// fragmentTestCache_.Decimate();
 
 	if (dumpNextFrame_) {

GPU/Directx9/GPU_DX9.h

@@ -24,6 +24,7 @@
 #include "GPU/Directx9/FramebufferDX9.h"
 #include "GPU/Directx9/TransformPipelineDX9.h"
 #include "GPU/Directx9/TextureCacheDX9.h"
+#include "GPU/Directx9/DepalettizeShaderDX9.h"
 #include "GPU/Directx9/helper/fbo.h"
 #include "GPU/Common/VertexDecoderCommon.h"
 
@@ -167,6 +168,7 @@ private:
 
 	FramebufferManagerDX9 framebufferManager_;
 	TextureCacheDX9 textureCache_;
+	DepalShaderCacheDX9 depalShaderCache_;
 	TransformDrawEngineDX9 transformDraw_;
 	ShaderManagerDX9 *shaderManager_;
 

GPU/Directx9/TextureCacheDX9.cpp

@@ -26,6 +26,8 @@
 #include "GPU/Directx9/PixelShaderGeneratorDX9.h"
 #include "GPU/Directx9/TextureCacheDX9.h"
 #include "GPU/Directx9/FramebufferDX9.h"
+#include "GPU/Directx9/ShaderManagerDX9.h"
+#include "GPU/Directx9/DepalettizeShaderDX9.h"
 #include "GPU/Directx9/helper/dx_state.h"
 #include "GPU/Common/FramebufferCommon.h"
 #include "GPU/Common/TextureDecoder.h"
@@ -897,21 +899,69 @@ void TextureCacheDX9::SetTextureFramebuffer(TexCacheEntry *entry, VirtualFramebu
 	framebuffer->usageFlags |= FB_USAGE_TEXTURE;
 	bool useBufferedRendering = g_Config.iRenderingMode != FB_NON_BUFFERED_MODE;
 	if (useBufferedRendering) {
-		// TODO: Depal
-		// For now, let's not bind FBOs that we know are off (invalidHint will be -1.)
-		// But let's still not use random memory.
-		if (entry->framebuffer->fbo) {
-			fbo_bind_color_as_texture(entry->framebuffer->fbo, 0);
-			// Keep the framebuffer alive.
-			// TODO: Dangerous if it sets a new one?
-			entry->framebuffer->last_frame_used = gpuStats.numFlips;
-		} else {
-			pD3Ddevice->SetTexture(0, NULL);
-			gstate_c.skipDrawReason |= SKIPDRAW_BAD_FB_TEXTURE;
+		LPDIRECT3DPIXELSHADER9 pshader = nullptr;
+		if ((entry->status & TexCacheEntry::STATUS_DEPALETTIZE) && !g_Config.bDisableSlowFramebufEffects) {
+			pshader = depalShaderCache_->GetDepalettizePixelShader(framebuffer->drawnFormat);
 		}
 
-		gstate_c.textureFullAlpha = gstate.getTextureFormat() == GE_TFMT_5650;
-		gstate_c.textureSimpleAlpha = gstate_c.textureFullAlpha;
+		if (pshader) {
+			LPDIRECT3DTEXTURE9 clutTexture = depalShaderCache_->GetClutTexture(clutHash_, clutBuf_);
+
+			FBO *depalFBO = framebufferManager_->GetTempFBO(framebuffer->renderWidth, framebuffer->renderHeight, FBO_8888);
+			fbo_bind_as_render_target(depalFBO);
+
+			static const float pos[12 + 8] = {
+				-1, -1, -1,  0, 0,
+				1, -1, -1,	   1, 0,
+				1, 1, -1,		 1, 1,
+				-1, 1, -1,		 0, 1,
+			};
+			static const u16 indices[4] = { 0, 1, 3, 2 };
+
+			shaderManager_->DirtyLastShader();
+
+			pD3Ddevice->SetPixelShader(pshader);
+			pD3Ddevice->SetVertexShader(depalShaderCache_->GetDepalettizeVertexShader());
+			pD3Ddevice->SetTexture(1, clutTexture);
+
+			framebufferManager_->BindFramebufferColor(0, framebuffer, true);
+
+			pD3Ddevice->SetRenderState(D3DRS_ZENABLE, FALSE);
+			pD3Ddevice->SetRenderState(D3DRS_STENCILENABLE, FALSE);
+			pD3Ddevice->SetRenderState(D3DRS_SCISSORTESTENABLE, FALSE);
+			pD3Ddevice->SetRenderState(D3DRS_CULLMODE, D3DCULL_NONE);
+			pD3Ddevice->SetFVF(D3DFVF_XYZ | D3DFVF_TEX0);
+
+			D3DVIEWPORT9 vp;
+			vp.MinZ = 0;
+			vp.MaxZ = 1;
+			vp.X = 0;
+			vp.Y = 0;
+			vp.Width = framebuffer->renderWidth;
+			vp.Height = framebuffer->renderHeight;
+			pD3Ddevice->SetViewport(&vp);
+
+			pD3Ddevice->DrawIndexedPrimitiveUP(D3DPT_TRIANGLEFAN, 0, 4, 2, indices, D3DFMT_INDEX16, pos, (3 + 2) * sizeof(float));
+
+			fbo_bind_color_as_texture(depalFBO, 0);
+			dxstate.Restore();
+			framebufferManager_->RebindFramebuffer();
+
+			const GEPaletteFormat clutFormat = gstate.getClutPaletteFormat();
+			const u32 clutBase = gstate.getClutIndexStartPos();
+			const u32 bytesPerColor = clutFormat == GE_CMODE_32BIT_ABGR8888 ? sizeof(u32) : sizeof(u16);
+			const u32 clutExtendedColors = (clutTotalBytes_ / bytesPerColor) + clutBase;
+
+			TexCacheEntry::Status alphaStatus = CheckAlpha(clutBuf_, getClutDestFormat(gstate.getClutPaletteFormat()), clutExtendedColors, clutExtendedColors, 1);
+			gstate_c.textureFullAlpha = alphaStatus == TexCacheEntry::STATUS_ALPHA_FULL;
+			gstate_c.textureSimpleAlpha = alphaStatus == TexCacheEntry::STATUS_ALPHA_SIMPLE;
+		} else {
+			entry->status &= ~TexCacheEntry::STATUS_DEPALETTIZE;
+			framebufferManager_->BindFramebufferColor(0, framebuffer, false);
+
+			gstate_c.textureFullAlpha = gstate.getTextureFormat() == GE_TFMT_5650;
+			gstate_c.textureSimpleAlpha = gstate_c.textureFullAlpha;
+		}
 
 		// Keep the framebuffer alive.
 		framebuffer->last_frame_used = gpuStats.numFlips;

GPU/Directx9/TextureCacheDX9.h

@@ -32,6 +32,7 @@ struct VirtualFramebuffer;
 namespace DX9 {
 
 class FramebufferManagerDX9;
+class DepalShaderCacheDX9;
 class ShaderManagerDX9;
 
 enum TextureFiltering {
@@ -69,6 +70,9 @@ public:
 	void SetFramebufferManager(FramebufferManagerDX9 *fbManager) {
 		framebufferManager_ = fbManager;
 	}
+	void SetDepalShaderCache(DepalShaderCacheDX9 *dpCache) {
+		depalShaderCache_ = dpCache;
+	}
 	void SetShaderManager(ShaderManagerDX9 *sm) {
 		shaderManager_ = sm;
 	}
@@ -223,6 +227,7 @@ private:
 	int timesInvalidatedAllThisFrame_;
 
 	FramebufferManagerDX9 *framebufferManager_;
+	DepalShaderCacheDX9 *depalShaderCache_;
 	ShaderManagerDX9 *shaderManager_;
 };