various improvements & cleanup for shader compilation

src/Cafe/HW/Latte/Core/LatteShader.cpp

@@ -357,19 +357,6 @@ void LatteShader_CreateRendererShader(LatteDecompilerShader* shader, bool compil
 	LatteShader_CleanupAfterCompile(shader);
 }
 
-void LatteShader_FinishCompilation(LatteDecompilerShader* shader)
-{
-	if (shader->hasError)
-	{
-		cemuLog_logDebug(LogType::Force, "LatteShader_finishCompilation(): Skipped because of error in shader {:x}", shader->baseHash);
-		return;
-	}
-	shader->shader->WaitForCompiled();
-
-	LatteShader_prepareSeparableUniforms(shader);
-	LatteShader_CleanupAfterCompile(shader);
-}
-
 void LatteSHRC_RegisterShader(LatteDecompilerShader* shader, uint64 baseHash, uint64 auxHash)
 {
 	auto& cache = LatteSHRC_GetCacheByType(shader->shaderType);
@@ -767,8 +754,8 @@ LatteDecompilerShader* LatteShader_CompileSeparableVertexShader(uint64 baseHash,
 	if (g_renderer->GetType() == RendererAPI::OpenGL)
 	{
 		if (vertexShader->shader)
-			vertexShader->shader->PreponeCompilation(true);
-		LatteShader_FinishCompilation(vertexShader);
+			vertexShader->shader->PreponeCompilation();
+		LatteShader_prepareSeparableUniforms(vertexShader);
 	}
 
 	LatteSHRC_RegisterShader(vertexShader, vertexShader->baseHash, vertexShader->auxHash);
@@ -796,8 +783,8 @@ LatteDecompilerShader* LatteShader_CompileSeparableGeometryShader(uint64 baseHas
 	if (g_renderer->GetType() == RendererAPI::OpenGL)
 	{
 		if (geometryShader->shader)
-			geometryShader->shader->PreponeCompilation(true);
-		LatteShader_FinishCompilation(geometryShader);
+			geometryShader->shader->PreponeCompilation();
+		LatteShader_prepareSeparableUniforms(geometryShader);
 	}
 
 	LatteSHRC_RegisterShader(geometryShader, geometryShader->baseHash, geometryShader->auxHash);
@@ -825,8 +812,8 @@ LatteDecompilerShader* LatteShader_CompileSeparablePixelShader(uint64 baseHash,
 	if (g_renderer->GetType() == RendererAPI::OpenGL)
 	{
 		if (pixelShader->shader)
-			pixelShader->shader->PreponeCompilation(true);
-		LatteShader_FinishCompilation(pixelShader);
+			pixelShader->shader->PreponeCompilation();
+		LatteShader_prepareSeparableUniforms(pixelShader);
 	}
 
 	LatteSHRC_RegisterShader(pixelShader, _shaderBaseHash_ps, psAuxHash);

src/Cafe/HW/Latte/Core/LatteShader.h

@@ -99,7 +99,6 @@ void LatteShader_GetDecompilerOptions(struct LatteDecompilerOptions& options, La
 LatteDecompilerShader* LatteShader_CreateShaderFromDecompilerOutput(LatteDecompilerOutput_t& decompilerOutput, uint64 baseHash, bool calculateAuxHash, uint64 optionalAuxHash, uint32* contextRegister);
 
 void LatteShader_CreateRendererShader(LatteDecompilerShader* shader, bool compileAsync);
-void LatteShader_FinishCompilation(LatteDecompilerShader* shader);
 
 void LatteShader_prepareSeparableUniforms(LatteDecompilerShader* shader);
 

src/Cafe/HW/Latte/Core/LatteShaderCache.cpp

@@ -102,13 +102,28 @@ void LatteShaderCache_removeFromCompileQueue(sint32 index)
  */
 void LatteShaderCache_updateCompileQueue(sint32 maxRemainingEntries)
 {
+	// remove any shaders that are already done
+	for (size_t i = 0; i < shaderCompileQueue.count; i++)
+	{
+		auto shaderEntry = shaderCompileQueue.entry[i].shader;
+		if (!shaderEntry)
+			continue;
+		if (shaderEntry->shader->IsCompiled())
+		{
+			LatteShader_prepareSeparableUniforms(shaderEntry);
+			LatteShaderCache_removeFromCompileQueue(i);
+		}
+	}
 	while (true)
 	{
 		if (shaderCompileQueue.count <= maxRemainingEntries)
 			break;
 		auto shader = shaderCompileQueue.entry[0].shader;
 		if (shader)
-			LatteShader_FinishCompilation(shader);
+		{
+			shader->shader->PreponeCompilation();
+			LatteShader_prepareSeparableUniforms(shader);
+		}
 		LatteShaderCache_removeFromCompileQueue(0);
 	}
 }

src/Cafe/HW/Latte/Core/LatteShaderGL.cpp

@@ -28,6 +28,8 @@ void LatteShader_prepareSeparableUniforms(LatteDecompilerShader* shader)
 {
 	if (g_renderer->GetType() == RendererAPI::Vulkan)
 		return;
+	if(shader->hasError)
+		return;
 
 	auto shaderGL = (RendererShaderGL*)shader->shader;
 	// setup uniform info

src/Cafe/HW/Latte/Renderer/OpenGL/OpenGLRendererCore.cpp

@@ -868,7 +868,7 @@ RendererShaderGL* rectsEmulationGS_generateShaderGL(LatteDecompilerShader* verte
 	gsSrc.append("}\r\n");
 
 	auto glShader = new RendererShaderGL(RendererShader::ShaderType::kGeometry, 0, 0, false, false, gsSrc);
-	glShader->PreponeCompilation(true);
+	glShader->PreponeCompilation();
 
 	return glShader;
 }

src/Cafe/HW/Latte/Renderer/OpenGL/RendererShaderGL.cpp

@@ -109,19 +109,8 @@ RendererShaderGL::RendererShaderGL(ShaderType type, uint64 baseHash, uint64 auxH
 	glShaderSource(m_shader_object, 1, &c_str, &size);
 	glCompileShader(m_shader_object);
 
-	GLint log_length;
-	glGetShaderiv(m_shader_object, GL_INFO_LOG_LENGTH, &log_length);
-	if (log_length > 0)
-	{
-		char log[2048]{};
-		GLsizei log_size;
-		glGetShaderInfoLog(m_shader_object, std::min<uint32>(log_length, sizeof(log) - 1), &log_size, log);
-		cemuLog_log(LogType::Force, "Error/Warning in shader:");
-		cemuLog_log(LogType::Force, log);
-	}
-
 	// set debug name
-	if (LaunchSettings::NSightModeEnabled()) 
+	if (LaunchSettings::NSightModeEnabled())
 	{
 		auto objNameStr = fmt::format("shader_{:016x}_{:016x}", m_baseHash, m_auxHash);
 		glObjectLabel(GL_SHADER, m_shader_object, objNameStr.size(), objNameStr.c_str());
@@ -134,12 +123,6 @@ RendererShaderGL::RendererShaderGL(ShaderType type, uint64 baseHash, uint64 auxH
 	m_shader_attached = true;
 	glLinkProgram(m_program);
 
-	storeBinary();
-
-	// count shader compilation
-	if (!s_isLoadingShaders)
-		++g_compiled_shaders_total;
-
 	// we can throw away the GLSL code to conserve RAM
 	m_glslSource.clear();
 	m_glslSource.shrink_to_fit();
@@ -157,7 +140,7 @@ RendererShaderGL::~RendererShaderGL()
 		glDeleteProgram(m_program);
 }
 
-void RendererShaderGL::PreponeCompilation(bool isRenderThread)
+void RendererShaderGL::PreponeCompilation()
 {
 	// the logic for initiating compilation is currently in the constructor
 	// here we only guarantee that it is finished before we return
@@ -168,56 +151,81 @@ void RendererShaderGL::PreponeCompilation(bool isRenderThread)
 
 bool RendererShaderGL::IsCompiled()
 {
-	cemu_assert_debug(false);
-	return true;
+	if(m_isCompiled)
+		return true;
+
+	if(!glMaxShaderCompilerThreadsARB)
+	{
+		WaitForCompiled();
+		return true;
+	}
+
+	GLint isShaderComplete = 0, isProgramComplete = 0;
+	glGetShaderiv(m_shader_object, GL_COMPLETION_STATUS_ARB, &isShaderComplete);
+	if(!isShaderComplete)
+		return false;
+	glGetProgramiv(m_program, GL_COMPLETION_STATUS_ARB, &isProgramComplete);
+	if (isProgramComplete)
+		WaitForCompiled(); // since COMPLETION_STATUS == true, this should be very fast
+	return m_isCompiled;
 }
 
 bool RendererShaderGL::WaitForCompiled()
 {
 	char infoLog[8 * 1024];
+	GLint log_length;
 	if (m_isCompiled)
 		return true;
+
+	// count shader compilation
+	if (!s_isLoadingShaders)
+		++g_compiled_shaders_total;
+
 	// check if compilation was successful
 	GLint compileStatus = GL_FALSE;
 	glGetShaderiv(m_shader_object, GL_COMPILE_STATUS, &compileStatus);
-	if (compileStatus == 0)
+	if (compileStatus == GL_FALSE)
 	{
-		uint32 infoLogLength, tempLength;
-		glGetShaderiv(m_shader_object, GL_INFO_LOG_LENGTH, (GLint *)&infoLogLength);
-		if (infoLogLength != 0)
+		glGetShaderiv(m_shader_object, GL_INFO_LOG_LENGTH, &log_length);
+		if (log_length > 0)
 		{
-			tempLength = sizeof(infoLog) - 1;
-			glGetShaderInfoLog(m_shader_object, std::min(infoLogLength, tempLength), (GLsizei*)&tempLength, (GLcharARB*)infoLog);
-			infoLog[tempLength] = '\0';
-			cemuLog_log(LogType::Force, "Compile error in shader. Log:");
+			glGetShaderInfoLog(m_shader_object, sizeof(infoLog), &log_length, infoLog);
+			cemuLog_log(LogType::Force, "Error/Warning in shader:");
 			cemuLog_log(LogType::Force, infoLog);
 		}
+
 		if (m_shader_object != 0)
 			glDeleteShader(m_shader_object);
+		if (m_program != 0)
+			glDeleteProgram(m_program);
+		m_program = m_shader_object = 0;
 		m_isCompiled = true;
 		return false;
 	}
+
 	// get shader binary
 	GLint linkStatus = GL_FALSE;
 	glGetProgramiv(m_program, GL_LINK_STATUS, &linkStatus);
-	if (linkStatus == 0)
+	if (linkStatus == GL_FALSE)
 	{
-		uint32 infoLogLength, tempLength;
-		glGetProgramiv(m_program, GL_INFO_LOG_LENGTH, (GLint *)&infoLogLength);
-		if (infoLogLength != 0)
+		glGetProgramiv(m_program, GL_INFO_LOG_LENGTH, (GLint *)&log_length);
+		if (log_length != 0)
 		{
-			tempLength = sizeof(infoLog) - 1;
-			glGetProgramInfoLog(m_program, std::min(infoLogLength, tempLength), (GLsizei*)&tempLength, (GLcharARB*)infoLog);
-			infoLog[tempLength] = '\0';
+			glGetProgramInfoLog(m_program, sizeof(infoLog), &log_length, (GLcharARB*)infoLog);
 			cemuLog_log(LogType::Force, "Link error in shader. Log:");
 			cemuLog_log(LogType::Force, infoLog);
 		}
 		m_isCompiled = true;
 		return false;
 	}
 
-	/*glDetachShader(m_program, m_shader_object);
-	m_shader_attached = false;*/
+	storeBinary();
+
+	glDetachShader(m_program, m_shader_object);
+	m_shader_attached = false;
+	glDeleteShader(m_shader_object);
+	m_shader_object = 0;
+
 	m_isCompiled = true;
 	return true;
 }
@@ -266,8 +274,8 @@ void RendererShaderGL::ShaderCacheLoading_begin(uint64 cacheTitleId)
 	{
 		const uint32 cacheMagic = GeneratePrecompiledCacheId();
 		const std::string cacheFilename = fmt::format("{:016x}_gl.bin", cacheTitleId);
-        s_programBinaryCache = FileCache::Open(ActiveSettings::GetCachePath("shaderCache/precompiled/{}", cacheFilename), true, cacheMagic);
-		if (s_programBinaryCache == nullptr)
+        s_programBinaryCache.reset(FileCache::Open(ActiveSettings::GetCachePath("shaderCache/precompiled/{}", cacheFilename), true, cacheMagic));
+		if (!s_programBinaryCache)
 			cemuLog_log(LogType::Force, "Unable to open OpenGL precompiled cache {}", cacheFilename);
 	}
 	s_isLoadingShaders = true;
@@ -280,13 +288,10 @@ void RendererShaderGL::ShaderCacheLoading_end()
 
 void RendererShaderGL::ShaderCacheLoading_Close()
 {
-    if(s_programBinaryCache)
-    {
-        delete s_programBinaryCache;
-        s_programBinaryCache = nullptr;
-    }
+	s_programBinaryCache.reset();
     g_compiled_shaders_total = 0;
     g_compiled_shaders_async = 0;
 }
 
-FileCache* RendererShaderGL::s_programBinaryCache{};
+
+std::unique_ptr<class FileCache> RendererShaderGL::s_programBinaryCache{};

src/Cafe/HW/Latte/Renderer/OpenGL/RendererShaderGL.h

@@ -10,7 +10,7 @@ class RendererShaderGL : public RendererShader
 
 	virtual ~RendererShaderGL();
 
-	void PreponeCompilation(bool isRenderThread) override;
+	void PreponeCompilation() override;
 	bool IsCompiled() override;
 	bool WaitForCompiled() override;
 
@@ -37,6 +37,6 @@ class RendererShaderGL : public RendererShader
 	bool m_shader_attached{ false };
 	bool m_isCompiled{ false };
 
-	static class FileCache* s_programBinaryCache;
+	static std::unique_ptr<class FileCache> s_programBinaryCache;
 };
 

src/Cafe/HW/Latte/Renderer/RendererOuputShader.cpp

@@ -121,8 +121,8 @@ RendererOutputShader::RendererOutputShader(const std::string& vertex_source, con
 	m_vertex_shader.reset(g_renderer->shader_create(RendererShader::ShaderType::kVertex, 0, 0, vertex_source, false, false));
 	m_fragment_shader.reset(g_renderer->shader_create(RendererShader::ShaderType::kFragment, 0, 0, finalFragmentSrc, false, false));
 
-	m_vertex_shader->PreponeCompilation(true);
-	m_fragment_shader->PreponeCompilation(true);
+	m_vertex_shader->PreponeCompilation();
+	m_fragment_shader->PreponeCompilation();
 
 	if (!m_vertex_shader->WaitForCompiled())
 		throw std::exception();

src/Cafe/HW/Latte/Renderer/RendererShader.h

@@ -14,7 +14,7 @@ class RendererShader
 
 	ShaderType GetType() const { return m_type; }
 
-	virtual void PreponeCompilation(bool isRenderThread) = 0; // if shader not yet compiled, compile it synchronously (if possible) or alternatively wait for compilation. After this function IsCompiled() is guaranteed to be true
+	virtual void PreponeCompilation() = 0; // if shader not yet compiled, compile it synchronously (if possible) or alternatively wait for compilation. After this function IsCompiled() is guaranteed to be true
 	virtual bool IsCompiled() = 0;
 	virtual bool WaitForCompiled() = 0;
 

src/Cafe/HW/Latte/Renderer/Vulkan/RendererShaderVk.cpp

@@ -174,7 +174,7 @@ class _ShaderVkThreadPool
 			job->m_compilationState.setValue(RendererShaderVk::COMPILATION_STATE::COMPILING);
 			s_compilationQueueMutex.unlock();
 			// compile
-			job->CompileInternal(false);
+			job->CompileInternal();
 			++g_compiled_shaders_async;
 			// mark as compiled
 			cemu_assert_debug(job->m_compilationState.getValue() == RendererShaderVk::COMPILATION_STATE::COMPILING);
@@ -280,7 +280,7 @@ void RendererShaderVk::FinishCompilation()
 	m_glslCode.shrink_to_fit();
 }
 
-void RendererShaderVk::CompileInternal(bool isRenderThread)
+void RendererShaderVk::CompileInternal()
 {
 	// try to retrieve SPIR-V module from cache
 	if (s_isLoadingShadersVk && (m_isGameShader && !m_isGfxPackShader) && s_spirvCache)
@@ -412,7 +412,7 @@ void RendererShaderVk::CompileInternal(bool isRenderThread)
 	FinishCompilation();
 }
 
-void RendererShaderVk::PreponeCompilation(bool isRenderThread)
+void RendererShaderVk::PreponeCompilation()
 {
 	ShaderVkThreadPool.s_compilationQueueMutex.lock();
 	bool isStillQueued = m_compilationState.hasState(COMPILATION_STATE::QUEUED);
@@ -432,7 +432,7 @@ void RendererShaderVk::PreponeCompilation(bool isRenderThread)
 	else
 	{
 		// compile synchronously
-		CompileInternal(isRenderThread);
+		CompileInternal();
 		m_compilationState.setValue(COMPILATION_STATE::DONE);
 	}
 }

src/Cafe/HW/Latte/Renderer/Vulkan/RendererShaderVk.h

@@ -52,12 +52,12 @@ class RendererShaderVk : public RendererShader
 		s_dependencyLock.unlock();
 	}
 
-	void PreponeCompilation(bool isRenderThread) override;
+	void PreponeCompilation() override;
 	bool IsCompiled() override;
 	bool WaitForCompiled() override;
 
 private:
-	void CompileInternal(bool isRenderThread);
+  void CompileInternal();
 
 	void FinishCompilation();
 

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanPipelineCompiler.cpp

@@ -159,7 +159,7 @@ RendererShaderVk* rectsEmulationGS_generate(LatteDecompilerShader* vertexShader,
 	gsSrc.append("}\r\n");
 
 	auto vkShader = new RendererShaderVk(RendererShader::ShaderType::kGeometry, 0, 0, false, false, gsSrc);
-	vkShader->PreponeCompilation(true);
+	vkShader->PreponeCompilation();
 	return vkShader;
 }
 
@@ -966,11 +966,11 @@ bool PipelineCompiler::Compile(bool forceCompile, bool isRenderThread, bool show
 	{
 		// if some shader stages are not compiled yet, compile them now
 		if (m_vkVertexShader && m_vkVertexShader->IsCompiled() == false)
-			m_vkVertexShader->PreponeCompilation(isRenderThread);
+			m_vkVertexShader->PreponeCompilation();
 		if (m_vkPixelShader && m_vkPixelShader->IsCompiled() == false)
-			m_vkPixelShader->PreponeCompilation(isRenderThread);
+			m_vkPixelShader->PreponeCompilation();
 		if (m_vkGeometryShader && m_vkGeometryShader->IsCompiled() == false)
-			m_vkGeometryShader->PreponeCompilation(isRenderThread);
+			m_vkGeometryShader->PreponeCompilation();
 	}
 
 	if (shaderStages.empty())