Vulkan: recreate libretro presentation when sync mask changes

Common/GPU/Vulkan/VulkanPresentation.h

@@ -18,6 +18,13 @@ public:
 
 	virtual void Destroy(VulkanContext *vulkan) = 0;
 
+	// Some presentation backends can change their image set while the device remains alive. The
+	// render manager checks this at a safe frame boundary before recording new work.
+	virtual bool NeedsRecreate() const { return false; }
+	// Called after the render manager has stopped submission, waited for the queue, and destroyed
+	// its backbuffer views. The backend must rebuild any images exposed through GetImage().
+	virtual bool Recreate(VulkanContext *) { return true; }
+
 	// Mirrors vkAcquireNextImageKHR: waits for the next image to become available, and signals
 	// signalSemaphore once it's safe to render into it.
 	virtual VkResult AcquireNextImage(VulkanContext *vulkan, VkSemaphore signalSemaphore, uint32_t *imageIndex) = 0;

Common/GPU/Vulkan/VulkanRenderManager.cpp

@@ -412,6 +412,22 @@ bool VulkanRenderManager::CreateSwapchainViewsAndDepth(VkCommandBuffer cmdInit,
 	return true;
 }
 
+bool VulkanRenderManager::RecreatePresentation() {
+	VulkanPresentation *presentation = vulkan_->GetPresentation();
+	if (!presentation || !presentation->NeedsRecreate()) {
+		return true;
+	}
+
+	DestroyBackbuffers();
+	// DestroyBackbuffers() queues its views for deletion. They must be gone before a presentation
+	// backend destroys the images those views reference.
+	vulkan_->PerformPendingDeletes();
+	if (!presentation->Recreate(vulkan_)) {
+		return false;
+	}
+	return CreateBackbuffers();
+}
+
 void VulkanRenderManager::StartThreads() {
 	{
 		std::unique_lock<std::mutex> lock(compileQueueMutex_);
@@ -719,6 +735,11 @@ void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfile
 	}
 	vkResetFences(device, 1, &frameData.fence);
 
+	if (!RecreatePresentation()) {
+		ERROR_LOG(Log::G3D, "Failed to recreate Vulkan presentation backbuffers");
+		return;
+	}
+
 	uint64_t frameId = frameIdGen_++;
 
 	PollPresentTiming();

Common/GPU/Vulkan/VulkanRenderManager.h

@@ -565,6 +565,7 @@ private:
 
 	void SanityCheckPassesOnAdd();
 	bool CreateSwapchainViewsAndDepth(VkCommandBuffer cmdInit, VulkanBarrierBatch *barriers, FrameDataShared &frameDataShared);
+	bool RecreatePresentation();
 
 	FrameDataShared frameDataShared_;
 

libretro/LibretroVulkanPresentation.cpp

@@ -8,15 +8,28 @@ LibretroVulkanPresentation::LibretroVulkanPresentation(retro_hw_render_interface
 	: vulkan_(vulkan), format_(format), extent_(extent) {
 }
 
-bool LibretroVulkanPresentation::Create(VulkanContext *context) {
-	dedicatedAllocation_ = context->Extensions().KHR_dedicated_allocation;
-
-	uint32_t mask = vulkan_->get_sync_index_mask(vulkan_->handle);
+uint32_t LibretroVulkanPresentation::ImageCountFromMask(uint32_t mask) {
 	uint32_t count = 0;
 	while (mask) {
 		count++;
 		mask >>= 1;
 	}
+	return count;
+}
+
+bool LibretroVulkanPresentation::IsValidImageIndex(uint32_t imageIndex) const {
+	return imageIndex < images_.size() && imageIndex < 32 && (syncIndexMask_ & (1u << imageIndex)) != 0;
+}
+
+bool LibretroVulkanPresentation::Create(VulkanContext *context) {
+	dedicatedAllocation_ = context->Extensions().KHR_dedicated_allocation;
+
+	syncIndexMask_ = vulkan_->get_sync_index_mask(vulkan_->handle);
+	uint32_t count = ImageCountFromMask(syncIndexMask_);
+	auto fail = [this, context]() {
+		Destroy(context);
+		return false;
+	};
 
 	VkDevice device = context->GetDevice();
 	images_.resize(count);
@@ -35,7 +48,7 @@ bool LibretroVulkanPresentation::Create(VulkanContext *context) {
 		info.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
 		info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
 		if (vkCreateImage(device, &info, nullptr, &img.image) != VK_SUCCESS) {
-			return false;
+			return fail();
 		}
 
 		VkMemoryRequirements memreq;
@@ -51,13 +64,13 @@ bool LibretroVulkanPresentation::Create(VulkanContext *context) {
 		}
 
 		if (!context->MemoryTypeFromProperties(memreq.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, &alloc.memoryTypeIndex)) {
-			return false;
+			return fail();
 		}
 		if (vkAllocateMemory(device, &alloc, nullptr, &img.memory) != VK_SUCCESS) {
-			return false;
+			return fail();
 		}
 		if (vkBindImageMemory(device, img.image, img.memory, 0) != VK_SUCCESS) {
-			return false;
+			return fail();
 		}
 
 		img.retroImage.create_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
@@ -69,7 +82,7 @@ bool LibretroVulkanPresentation::Create(VulkanContext *context) {
 		img.retroImage.create_info.subresourceRange.layerCount = 1;
 		img.retroImage.create_info.subresourceRange.levelCount = 1;
 		if (vkCreateImageView(device, &img.retroImage.create_info, nullptr, &img.retroImage.image_view) != VK_SUCCESS) {
-			return false;
+			return fail();
 		}
 		img.retroImage.image_layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 	}
@@ -91,17 +104,39 @@ void LibretroVulkanPresentation::Destroy(VulkanContext *context) {
 		}
 	}
 	images_.clear();
+	syncIndexMask_ = 0;
+	std::lock_guard<std::mutex> lock(mutex_);
+	currentIndex_ = -1;
+	everPresented_ = false;
+}
+
+bool LibretroVulkanPresentation::NeedsRecreate() const {
+	return vulkan_->get_sync_index_mask(vulkan_->handle) != syncIndexMask_;
+}
+
+bool LibretroVulkanPresentation::Recreate(VulkanContext *context) {
+	Destroy(context);
+	return Create(context);
 }
 
 VkResult LibretroVulkanPresentation::AcquireNextImage(VulkanContext *vulkan, VkSemaphore signalSemaphore, uint32_t *imageIndex) {
 	// Unlike a real swapchain, RetroArch doesn't signal signalSemaphore for us here - PrepareSubmit()
 	// strips any wait on it before the real vkQueueSubmit, matching the old hack's behavior.
+	if (vulkan_->get_sync_index_mask(vulkan_->handle) != syncIndexMask_) {
+		return VK_ERROR_OUT_OF_DATE_KHR;
+	}
 	vulkan_->wait_sync_index(vulkan_->handle);
 	*imageIndex = vulkan_->get_sync_index(vulkan_->handle);
+	if (!IsValidImageIndex(*imageIndex)) {
+		return VK_ERROR_OUT_OF_DATE_KHR;
+	}
 	return VK_SUCCESS;
 }
 
 VkResult LibretroVulkanPresentation::QueuePresent(VulkanContext *vulkan, VkQueue queue, uint32_t imageIndex, VkSemaphore waitSemaphore) {
+	if (vulkan_->get_sync_index_mask(vulkan_->handle) != syncIndexMask_ || !IsValidImageIndex(imageIndex)) {
+		return VK_ERROR_OUT_OF_DATE_KHR;
+	}
 	std::unique_lock<std::mutex> lock(mutex_);
 	currentIndex_ = (int)imageIndex;
 	vulkan_->set_image(vulkan_->handle, &images_[imageIndex].retroImage, 0, nullptr, vulkan_->queue_index);

libretro/LibretroVulkanPresentation.h

@@ -25,6 +25,8 @@ public:
 
 	bool Create(VulkanContext *context);
 	void Destroy(VulkanContext *context) override;
+	bool NeedsRecreate() const override;
+	bool Recreate(VulkanContext *context) override;
 
 	VkResult AcquireNextImage(VulkanContext *vulkan, VkSemaphore signalSemaphore, uint32_t *imageIndex) override;
 	VkResult QueuePresent(VulkanContext *vulkan, VkQueue queue, uint32_t imageIndex, VkSemaphore waitSemaphore) override;
@@ -57,6 +59,7 @@ private:
 	VkFormat format_;
 	VkExtent2D extent_;
 	bool dedicatedAllocation_ = false;
+	uint32_t syncIndexMask_ = 0;
 
 	std::vector<Image> images_;
 
@@ -64,4 +67,7 @@ private:
 	std::condition_variable condVar_;
 	int currentIndex_ = -1;
 	bool everPresented_ = false;
+
+	static uint32_t ImageCountFromMask(uint32_t mask);
+	bool IsValidImageIndex(uint32_t imageIndex) const;
 };