From 3efb87ba6ded05f816daba5a969b54d79c03e34d Mon Sep 17 00:00:00 2001 From: Flyinghead Date: Mon, 11 Nov 2024 22:15:57 +0100 Subject: [PATCH] vk: fix crash when enabling/disabling EmulateFramebuffer Make sure to end the current render pass with the previous setting. Fix initial layout of color attachments in OIT. Fix missing initial transition after Term/Init in !OIT. Issue #1734 --- core/rend/vulkan/drawer.cpp | 5 +++-- core/rend/vulkan/drawer.h | 3 +++ core/rend/vulkan/oit/oit_drawer.cpp | 5 ++--- core/rend/vulkan/oit/oit_drawer.h | 11 ++++++++++- core/rend/vulkan/oit/oit_renderer.cpp | 20 +++++++++++--------- core/rend/vulkan/vulkan_renderer.cpp | 20 +++++++++++--------- 6 files changed, 40 insertions(+), 24 deletions(-) diff --git a/core/rend/vulkan/drawer.cpp b/core/rend/vulkan/drawer.cpp index fd6220bc60..c6baa42510 100644 --- a/core/rend/vulkan/drawer.cpp +++ b/core/rend/vulkan/drawer.cpp @@ -636,6 +636,7 @@ void TextureDrawer::EndRenderPass() void ScreenDrawer::Init(SamplerManager *samplerManager, ShaderManager *shaderManager, const vk::Extent2D& viewport) { + emulateFramebuffer = config::EmulateFramebuffer; this->shaderManager = shaderManager; if (this->viewport != viewport) { @@ -747,7 +748,7 @@ vk::CommandBuffer ScreenDrawer::BeginRenderPass() { setImageLayout(commandBuffer, colorAttachments[GetCurrentImage()]->GetImage(), vk::Format::eR8G8B8A8Unorm, 1, vk::ImageLayout::eUndefined, - config::EmulateFramebuffer ? vk::ImageLayout::eTransferSrcOptimal : vk::ImageLayout::eShaderReadOnlyOptimal); + emulateFramebuffer ? vk::ImageLayout::eTransferSrcOptimal : vk::ImageLayout::eShaderReadOnlyOptimal); transitionNeeded[GetCurrentImage()] = false; } @@ -774,7 +775,7 @@ void ScreenDrawer::EndRenderPass() if (!renderPassStarted) return; currentCommandBuffer.endRenderPass(); - if (config::EmulateFramebuffer) + if (emulateFramebuffer) { scaleAndWriteFramebuffer(currentCommandBuffer, colorAttachments[GetCurrentImage()].get()); } diff --git a/core/rend/vulkan/drawer.h b/core/rend/vulkan/drawer.h index a22867ed8b..5fe86ed8ea 100644 --- a/core/rend/vulkan/drawer.h +++ b/core/rend/vulkan/drawer.h @@ -298,6 +298,8 @@ class ScreenDrawer : public Drawer framebuffers.clear(); colorAttachments.clear(); depthAttachment.reset(); + transitionNeeded.clear(); + clearNeeded.clear(); Drawer::Term(); } @@ -333,6 +335,7 @@ class ScreenDrawer : public Drawer std::vector clearNeeded; bool frameRendered = false; float aspectRatio = 0.f; + bool emulateFramebuffer = false; }; class TextureDrawer : public Drawer diff --git a/core/rend/vulkan/oit/oit_drawer.cpp b/core/rend/vulkan/oit/oit_drawer.cpp index 2695ad3eba..bbcb2ae58c 100644 --- a/core/rend/vulkan/oit/oit_drawer.cpp +++ b/core/rend/vulkan/oit/oit_drawer.cpp @@ -296,9 +296,8 @@ bool OITDrawer::Draw(const Texture *fogTexture, const Texture *paletteTexture) needAttachmentTransition = false; // Not convinced that this is really needed but it makes validation layers happy for (auto& attachment : colorAttachments) - // FIXME should be eTransferSrcOptimal if fullFB (screen) or copy to vram (rtt) -> 1 validation error at startup setImageLayout(cmdBuffer, attachment->GetImage(), vk::Format::eR8G8B8A8Unorm, 1, - vk::ImageLayout::eUndefined, vk::ImageLayout::eShaderReadOnlyOptimal); + vk::ImageLayout::eUndefined, getAttachmentInitialLayout()); for (auto& attachment : depthAttachments) setImageLayout(cmdBuffer, attachment->GetImage(), GetContext()->GetDepthFormat(), 1, vk::ImageLayout::eUndefined, vk::ImageLayout::eDepthStencilReadOnlyOptimal); @@ -556,7 +555,7 @@ void OITScreenDrawer::MakeFramebuffers(const vk::Extent2D& viewport) // make sure all attachments have the same dimensions maxWidth = 0; maxHeight = 0; - MakeBuffers(viewport.width, viewport.height, config::EmulateFramebuffer ? vk::ImageUsageFlagBits::eTransferSrc : vk::ImageUsageFlagBits::eSampled); + MakeBuffers(viewport.width, viewport.height, emulateFramebuffer ? vk::ImageUsageFlagBits::eTransferSrc : vk::ImageUsageFlagBits::eSampled); clearNeeded = { true, true }; } diff --git a/core/rend/vulkan/oit/oit_drawer.h b/core/rend/vulkan/oit/oit_drawer.h index 5b9e0536b4..7075351bf6 100644 --- a/core/rend/vulkan/oit/oit_drawer.h +++ b/core/rend/vulkan/oit/oit_drawer.h @@ -83,6 +83,7 @@ class OITDrawer : public BaseDrawer void MakeBuffers(int width, int height, vk::ImageUsageFlags colorUsage = {}); virtual vk::Framebuffer getFramebuffer(int renderPass, int renderPassCount) = 0; int getFramebufferIndex() { return framebufferIndex; } + virtual vk::ImageLayout getAttachmentInitialLayout() = 0; vk::Rect2D viewport; std::array, 2> colorAttachments; @@ -138,6 +139,7 @@ class OITScreenDrawer : public OITDrawer void Init(SamplerManager *samplerManager, OITShaderManager *shaderManager, OITBuffers *oitBuffers, const vk::Extent2D& viewport) { + emulateFramebuffer = config::EmulateFramebuffer; if (!screenPipelineManager) { screenPipelineManager = std::make_unique(); screenPipelineManager->Init(shaderManager, oitBuffers); @@ -160,7 +162,7 @@ class OITScreenDrawer : public OITDrawer if (!frameStarted) return; frameStarted = false; - if (config::EmulateFramebuffer) { + if (emulateFramebuffer) { scaleAndWriteFramebuffer(currentCommandBuffer, colorAttachments[framebufferIndex].get()); } else @@ -189,6 +191,9 @@ class OITScreenDrawer : public OITDrawer protected: vk::Framebuffer getFramebuffer(int renderPass, int renderPassCount) override; + vk::ImageLayout getAttachmentInitialLayout() override { + return emulateFramebuffer ? vk::ImageLayout::eTransferSrcOptimal : vk::ImageLayout::eShaderReadOnlyOptimal; + } private: void MakeFramebuffers(const vk::Extent2D& viewport); @@ -197,6 +202,7 @@ class OITScreenDrawer : public OITDrawer bool frameRendered = false; float aspectRatio = 0.f; bool frameStarted = false; + bool emulateFramebuffer = false; }; class OITTextureDrawer : public OITDrawer @@ -225,6 +231,9 @@ class OITTextureDrawer : public OITDrawer protected: vk::CommandBuffer NewFrame() override; vk::Framebuffer getFramebuffer(int renderPass, int renderPassCount) override; + vk::ImageLayout getAttachmentInitialLayout() override { + return config::RenderToTextureBuffer ? vk::ImageLayout::eTransferSrcOptimal : vk::ImageLayout::eShaderReadOnlyOptimal; + } private: u32 textureAddr = 0; diff --git a/core/rend/vulkan/oit/oit_renderer.cpp b/core/rend/vulkan/oit/oit_renderer.cpp index a8d3d37c06..e5e0bc7e05 100644 --- a/core/rend/vulkan/oit/oit_renderer.cpp +++ b/core/rend/vulkan/oit/oit_renderer.cpp @@ -64,22 +64,24 @@ class OITVulkanRenderer final : public BaseVulkanRenderer void Process(TA_context* ctx) override { - if (ctx->rend.isRTT) + if (emulateFramebuffer != config::EmulateFramebuffer) + { + screenDrawer.EndFrame(); + VulkanContext::Instance()->WaitIdle(); + screenDrawer.Term(); + screenDrawer.Init(&samplerManager, &oitShaderManager, &oitBuffers, viewport); + BaseInit(screenDrawer.GetRenderPass(), 2); + emulateFramebuffer = config::EmulateFramebuffer; + } + else if (ctx->rend.isRTT) { screenDrawer.EndFrame(); + } BaseVulkanRenderer::Process(ctx); } bool Render() override { try { - if (emulateFramebuffer != config::EmulateFramebuffer) - { - VulkanContext::Instance()->WaitIdle(); - screenDrawer.Term(); - screenDrawer.Init(&samplerManager, &oitShaderManager, &oitBuffers, viewport); - BaseInit(screenDrawer.GetRenderPass(), 2); - emulateFramebuffer = config::EmulateFramebuffer; - } OITDrawer *drawer; if (pvrrc.isRTT) drawer = &textureDrawer; diff --git a/core/rend/vulkan/vulkan_renderer.cpp b/core/rend/vulkan/vulkan_renderer.cpp index db4294db16..27707cc433 100644 --- a/core/rend/vulkan/vulkan_renderer.cpp +++ b/core/rend/vulkan/vulkan_renderer.cpp @@ -338,22 +338,24 @@ class VulkanRenderer final : public BaseVulkanRenderer void Process(TA_context* ctx) override { - if (ctx->rend.isRTT) + if (emulateFramebuffer != config::EmulateFramebuffer) + { + screenDrawer.EndRenderPass(); + VulkanContext::Instance()->WaitIdle(); + screenDrawer.Term(); + screenDrawer.Init(&samplerManager, &shaderManager, viewport); + BaseInit(screenDrawer.GetRenderPass()); + emulateFramebuffer = config::EmulateFramebuffer; + } + else if (ctx->rend.isRTT) { screenDrawer.EndRenderPass(); + } BaseVulkanRenderer::Process(ctx); } bool Render() override { try { - if (emulateFramebuffer != config::EmulateFramebuffer) - { - VulkanContext::Instance()->WaitIdle(); - screenDrawer.Term(); - screenDrawer.Init(&samplerManager, &shaderManager, viewport); - BaseInit(screenDrawer.GetRenderPass()); - emulateFramebuffer = config::EmulateFramebuffer; - } Drawer *drawer; if (pvrrc.isRTT) drawer = &textureDrawer;