From d47a457cca498aae7521c6fbdf7e7cb830c888e1 Mon Sep 17 00:00:00 2001
From: Jasmine Hansen <jasmine.hansen@ntd.nintendo.com>
Date: Fri, 17 Jan 2025 14:03:22 -0800
Subject: [PATCH] Fix render overlay dynamic state for VK_EXT_shader_object

Mirrors the pipeline modifications with dynamic state for Highlight Draw, Depth, Stencil, Backface Cull, and Viewport Scissor
---
 renderdoc/driver/vulkan/vk_overlay.cpp | 54 +++++++++++++++++++++++++-
 1 file changed, 53 insertions(+), 1 deletion(-)

diff --git a/renderdoc/driver/vulkan/vk_overlay.cpp b/renderdoc/driver/vulkan/vk_overlay.cpp
index 8d3c5523e8..ed64314019 100644
--- a/renderdoc/driver/vulkan/vk_overlay.cpp
+++ b/renderdoc/driver/vulkan/vk_overlay.cpp
@@ -1080,6 +1080,8 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, D
       state.depthBoundsTestEnable = VK_FALSE;
       state.cullMode = VK_CULL_MODE_NONE;
 
+      state.sampleMask = {~0U};
+
       // disable all discard rectangles
       RemoveNextStruct(&pipeCreateInfo,
                        VK_STRUCTURE_TYPE_PIPELINE_DISCARD_RECTANGLE_STATE_CREATE_INFO_EXT);
@@ -1178,6 +1180,12 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, D
         }
       }
 
+      state.logicOpEnable = false;
+      for(uint32_t i = 0; i < state.colorBlendEnable.size(); i++)
+        state.colorBlendEnable[i] = false;
+      for(uint32_t i = 0; i < state.colorWriteMask.size(); i++)
+        state.colorWriteMask[i] = 0xf;
+
       // set scissors to max for drawcall
       if(overlay == DebugOverlay::Drawcall && pipeCreateInfo.pViewportState)
       {
@@ -1488,10 +1496,18 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, D
       state.depthBoundsTestEnable = VK_FALSE;
       state.cullMode = VK_CULL_MODE_NONE;
 
+      state.sampleMask = {~0U};
+
       // enable dynamic depth clamp
       if(m_pDriver->GetDeviceEnabledFeatures().depthClamp)
         state.depthClampEnable = true;
 
+      state.logicOpEnable = false;
+      for(uint32_t i = 0; i < state.colorBlendEnable.size(); i++)
+        state.colorBlendEnable[i] = false;
+      for(uint32_t i = 0; i < state.colorWriteMask.size(); i++)
+        state.colorWriteMask[i] = 0xf;
+
       // modify state
       state.SetRenderPass(GetResID(m_Overlay.NoDepthRP));
       state.subpass = 0;
@@ -1817,10 +1833,18 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, D
       state.depthBoundsTestEnable = VK_FALSE;
       state.cullMode = VK_CULL_MODE_NONE;
 
+      state.sampleMask = {~0U};
+
       // enable dynamic depth clamp
       if(m_pDriver->GetDeviceEnabledFeatures().depthClamp)
         state.depthClampEnable = true;
 
+      state.logicOpEnable = false;
+      for(uint32_t i = 0; i < state.colorBlendEnable.size(); i++)
+        state.colorBlendEnable[i] = false;
+      for(uint32_t i = 0; i < state.colorWriteMask.size(); i++)
+        state.colorWriteMask[i] = 0xf;
+
       // modify state
       state.SetRenderPass(GetResID(m_Overlay.NoDepthRP));
       state.subpass = 0;
@@ -1953,7 +1977,9 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, D
         if(useDepthWriteStencilPass)
         {
           useDepthWriteStencilPass = false;
-          const VulkanCreationInfo::ShaderEntry &ps = pipeInfo.shaders[4];
+          const VulkanCreationInfo::ShaderEntry &ps =
+              state.graphics.shaderObject ? createinfo.m_ShaderObject[state.shaderObjects[4]].shad
+                                          : pipeInfo.shaders[4];
           if(ps.module != ResourceId())
           {
             ShaderReflection *reflection = ps.refl;
@@ -2473,10 +2499,32 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, D
       state.depthBoundsTestEnable = VK_FALSE;
       state.cullMode = VK_CULL_MODE_NONE;
 
+      state.sampleMask = {~0U};
+
       // enable dynamic depth clamp
       if(m_pDriver->GetDeviceEnabledFeatures().depthClamp)
         state.depthClampEnable = true;
 
+      state.logicOpEnable = false;
+      for(uint32_t i = 0; i < state.colorBlendEnable.size(); i++)
+        state.colorBlendEnable[i] = false;
+      for(uint32_t i = 0; i < state.colorWriteMask.size(); i++)
+        state.colorWriteMask[i] = 0xf;
+
+      if(depthRP != VK_NULL_HANDLE)
+      {
+        if(overlay == DebugOverlay::Depth)
+        {
+          state.depthTestEnable = origDepthTest;
+        }
+        else
+        {
+          state.front.passOp = state.front.failOp = state.front.depthFailOp = VK_STENCIL_OP_KEEP;
+          state.back.passOp = state.back.failOp = state.back.depthFailOp = VK_STENCIL_OP_KEEP;
+          state.stencilTestEnable = origStencilTest;
+        }
+      }
+
       if(state.graphics.shaderObject)
       {
         state.graphics.pipeline = ResourceId();
@@ -2487,6 +2535,10 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, D
 
       if(useDepthWriteStencilPass)
       {
+        // disable colour write
+        for(uint32_t i = 0; i < state.colorWriteMask.size(); i++)
+          state.colorWriteMask[i] = 0x0;
+
         // override stencil dynamic state
         state.front.compare = 0xff;
         state.front.write = 0xff;