Record setup command buffer in a background thread.

src/engine/renderer/vulkan/renderer_vulkan.cc

@@ -206,12 +206,21 @@ void RendererVulkan::CreateGeometry(std::shared_ptr<void> impl_data,
                      VK_BUFFER_USAGE_VERTEX_BUFFER_BIT,
                  VMA_MEMORY_USAGE_GPU_ONLY);
 
-  UpdateBuffer(std::get<0>(geometry->buffer), 0, mesh->GetVertices(),
+  task_runner_.PostTask(HERE, std::bind(&RendererVulkan::UpdateBuffer, this,
-               data_size);
+                                        std::get<0>(geometry->buffer), 0,
-  BufferMemoryBarrier(
+                                        mesh->GetVertices(), data_size));
+  task_runner_.PostTask(HERE,
+                        std::bind(&RendererVulkan::BufferMemoryBarrier, this,
                                   std::get<0>(geometry->buffer), 0, data_size,
-      VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
+                                  VK_PIPELINE_STAGE_TRANSFER_BIT,
-      VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT);
+                                  VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
+                                  VK_ACCESS_TRANSFER_WRITE_BIT,
+                                  VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT));
+  task_runner_.PostTask(HERE, [&, mesh = mesh.release()]() {
+    // Transfer mesh ownership to the background thread.
+    std::unique_ptr<Mesh> own(mesh);
+  });
+  semaphore_.Release();
 }
 
 void RendererVulkan::DestroyGeometry(std::shared_ptr<void> impl_data) {
@@ -253,19 +262,31 @@ void RendererVulkan::UpdateTexture(std::shared_ptr<void> impl_data,
     texture->height = image->GetHeight();
   }
 
-  ImageMemoryBarrier(
+  task_runner_.PostTask(
-      std::get<0>(texture->image), VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
+      HERE,
+      std::bind(&RendererVulkan::ImageMemoryBarrier, this,
+                std::get<0>(texture->image),
+                VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
                 VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT,
-      old_layout, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
+                old_layout, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL));
-  UpdateImage(std::get<0>(texture->image), image->GetBuffer(),
+  task_runner_.PostTask(
-              image->GetWidth(), image->GetHeight());
+      HERE,
-  ImageMemoryBarrier(std::get<0>(texture->image), VK_ACCESS_TRANSFER_WRITE_BIT,
+      std::bind(&RendererVulkan::UpdateImage, this, std::get<0>(texture->image),
+                image->GetBuffer(), image->GetWidth(), image->GetHeight()));
+  task_runner_.PostTask(
+      HERE, std::bind(&RendererVulkan::ImageMemoryBarrier, this,
+                      std::get<0>(texture->image), VK_ACCESS_TRANSFER_WRITE_BIT,
                       VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
                           VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
                           VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                       0, VK_ACCESS_SHADER_READ_BIT,
                       VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-                     VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+                      VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL));
+  task_runner_.PostTask(HERE, [&, image = image.release()]() {
+    // Transfer image ownership to the background thread.
+    std::unique_ptr<Image> own(image);
+  });
+  semaphore_.Release();
 }
 
 void RendererVulkan::DestroyTexture(std::shared_ptr<void> impl_data) {
@@ -569,10 +590,17 @@ bool RendererVulkan::InitializeInternal() {
     cmd_pool_info.queueFamilyIndex = context_.GetGraphicsQueue();
     cmd_pool_info.flags = 0;
 
-    VkResult res = vkCreateCommandPool(device_, &cmd_pool_info, nullptr,
+    VkResult err = vkCreateCommandPool(device_, &cmd_pool_info, nullptr,
-                                       &frames_[i].command_pool);
+                                       &frames_[i].setup_command_pool);
-    if (res) {
+    if (err) {
-      DLOG << "vkCreateCommandPool failed with error " << std::to_string(res);
+      DLOG << "vkCreateCommandPool failed with error " << std::to_string(err);
+      return false;
+    }
+
+    err = vkCreateCommandPool(device_, &cmd_pool_info, nullptr,
+                              &frames_[i].draw_command_pool);
+    if (err) {
+      DLOG << "vkCreateCommandPool failed with error " << std::to_string(err);
       return false;
     }
 
@@ -580,11 +608,11 @@ bool RendererVulkan::InitializeInternal() {
     VkCommandBufferAllocateInfo cmdbuf_info;
     cmdbuf_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
     cmdbuf_info.pNext = nullptr;
-    cmdbuf_info.commandPool = frames_[i].command_pool;
+    cmdbuf_info.commandPool = frames_[i].setup_command_pool;
     cmdbuf_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
     cmdbuf_info.commandBufferCount = 1;
 
-    VkResult err = vkAllocateCommandBuffers(device_, &cmdbuf_info,
+    err = vkAllocateCommandBuffers(device_, &cmdbuf_info,
                                    &frames_[i].setup_command_buffer);
     if (err) {
       DLOG << "vkAllocateCommandBuffers failed with error "
@@ -592,6 +620,7 @@ bool RendererVulkan::InitializeInternal() {
       continue;
     }
 
+    cmdbuf_info.commandPool = frames_[i].draw_command_pool;
     err = vkAllocateCommandBuffers(device_, &cmdbuf_info,
                                    &frames_[i].draw_command_buffer);
     if (err) {
@@ -604,17 +633,6 @@ bool RendererVulkan::InitializeInternal() {
   // Begin the first command buffer for the first frame.
   BeginFrame();
 
-  if (max_staging_buffer_size_ < staging_buffer_size_ * 4)
-    max_staging_buffer_size_ = staging_buffer_size_ * 4;
-
-  current_staging_buffer_ = 0;
-  staging_buffer_used_ = false;
-
-  for (int i = 0; i < frame_count; i++) {
-    bool err = InsertStagingBuffer();
-    LOG_IF(!err) << "Failed to create staging buffer.";
-  }
-
   // In this simple engine we use only one descriptor set that is for textures.
   // We use push contants for everything else.
   VkDescriptorSetLayoutBinding ds_layout_binding;
@@ -666,6 +684,11 @@ bool RendererVulkan::InitializeInternal() {
     return false;
   }
 
+  // Use a background thread for filling up staging buffers and recording setup
+  // commands.
+  setup_thread_ =
+      std::thread(&RendererVulkan::SetupThreadMain, this, frame_count);
+
   return true;
 }
 
@@ -673,15 +696,16 @@ void RendererVulkan::Shutdown() {
   LOG << "Shutting down renderer.";
   vkDeviceWaitIdle(device_);
 
+  quit_.store(true, std::memory_order_relaxed);
+  semaphore_.Release();
+  setup_thread_.join();
+
   for (int i = 0; i < frames_.size(); ++i) {
     FreePendingResources(i);
-    vkDestroyCommandPool(device_, frames_[i].command_pool, nullptr);
+    vkDestroyCommandPool(device_, frames_[i].setup_command_pool, nullptr);
+    vkDestroyCommandPool(device_, frames_[i].draw_command_pool, nullptr);
   }
 
-  for (int i = 0; i < staging_buffers_.size(); i++) {
-    auto [buffer, allocation] = staging_buffers_[i].buffer;
-    vmaDestroyBuffer(allocator_, buffer, allocation);
-  }
   vmaDestroyAllocator(allocator_);
 
   vkDestroyDescriptorSetLayout(device_, descriptor_set_layout_, nullptr);
@@ -697,7 +721,8 @@ void RendererVulkan::Shutdown() {
 void RendererVulkan::BeginFrame() {
   FreePendingResources(current_frame_);
 
-  vkResetCommandPool(device_, frames_[current_frame_].command_pool, 0);
+  vkResetCommandPool(device_, frames_[current_frame_].setup_command_pool, 0);
+  vkResetCommandPool(device_, frames_[current_frame_].draw_command_pool, 0);
 
   VkCommandBufferBeginInfo cmdbuf_begin;
   cmdbuf_begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
@@ -732,13 +757,12 @@ void RendererVulkan::BeginFrame() {
   }
 }
 
-void RendererVulkan::Flush() {
+void RendererVulkan::FlushSetupBuffer() {
   vkEndCommandBuffer(frames_[current_frame_].setup_command_buffer);
-  vkEndCommandBuffer(frames_[current_frame_].draw_command_buffer);
 
-  context_.Flush();
+  context_.Flush(false);
 
-  vkResetCommandPool(device_, frames_[current_frame_].command_pool, 0);
+  vkResetCommandPool(device_, frames_[current_frame_].setup_command_pool, 0);
 
   VkCommandBufferBeginInfo cmdbuf_begin;
   cmdbuf_begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
@@ -752,15 +776,8 @@ void RendererVulkan::Flush() {
     DLOG << "vkBeginCommandBuffer failed with error " << std::to_string(err);
     return;
   }
-  context_.AppendCommandBuffer(frames_[current_frame_].setup_command_buffer);
+  context_.AppendCommandBuffer(frames_[current_frame_].setup_command_buffer,
-
+                               true);
-  err = vkBeginCommandBuffer(frames_[current_frame_].draw_command_buffer,
-                             &cmdbuf_begin);
-  if (err) {
-    DLOG << "vkBeginCommandBuffer failed with error " << std::to_string(err);
-    return;
-  }
-  context_.AppendCommandBuffer(frames_[current_frame_].draw_command_buffer);
 }
 
 void RendererVulkan::FreePendingResources(int frame) {
@@ -880,7 +897,7 @@ bool RendererVulkan::AllocateStagingBuffer(uint32_t amount,
           } else {
             // Worst case scenario, all the staging buffers belong to this frame
             // and this frame is not even done. Flush everything.
-            Flush();
+            FlushSetupBuffer();
 
             // Clear the whole staging buffer.
             for (int i = 0; i < staging_buffers_.size(); i++) {
@@ -1080,7 +1097,7 @@ void RendererVulkan::FreeBuffer(Buffer<VkBuffer> buffer) {
   frames_[current_frame_].buffers_to_destroy.push_back(std::move(buffer));
 }
 
-bool RendererVulkan::UpdateBuffer(VkBuffer buffer,
+void RendererVulkan::UpdateBuffer(VkBuffer buffer,
                                   size_t offset,
                                   const void* data,
                                   size_t data_size) {
@@ -1094,7 +1111,7 @@ bool RendererVulkan::UpdateBuffer(VkBuffer buffer,
     if (!AllocateStagingBuffer(
             std::min((uint32_t)to_submit, staging_buffer_size_), 32,
             block_write_offset, block_write_amount))
-      return false;
+      return;
     Buffer<VkBuffer> staging_buffer =
         staging_buffers_[current_staging_buffer_].buffer;
 
@@ -1116,7 +1133,6 @@ bool RendererVulkan::UpdateBuffer(VkBuffer buffer,
     to_submit -= block_write_amount;
     submit_from += block_write_amount;
   }
-  return true;
 }
 
 void RendererVulkan::BufferMemoryBarrier(VkBuffer buffer,
@@ -1267,7 +1283,7 @@ void RendererVulkan::FreeTexture(Buffer<VkImage> image,
   frames_[current_frame_].desc_sets_to_destroy.push_back(std::move(desc_set));
 }
 
-bool RendererVulkan::UpdateImage(VkImage image,
+void RendererVulkan::UpdateImage(VkImage image,
                                  const uint8_t* data,
                                  int width,
                                  int height) {
@@ -1286,7 +1302,7 @@ bool RendererVulkan::UpdateImage(VkImage image,
 
     if (!AllocateStagingBuffer(std::min((uint32_t)to_submit, max_size), segment,
                                block_write_offset, block_write_amount))
-      return false;
+      return;
     Buffer<VkBuffer> staging_buffer =
         staging_buffers_[current_staging_buffer_].buffer;
 
@@ -1321,10 +1337,9 @@ bool RendererVulkan::UpdateImage(VkImage image,
     to_submit -= block_write_amount;
     submit_from += block_write_amount;
   }
-  return true;
 }
 
-void RendererVulkan::ImageMemoryBarrier(VkImage& image,
+void RendererVulkan::ImageMemoryBarrier(VkImage image,
                                         VkPipelineStageFlags src_stage_mask,
                                         VkPipelineStageFlags dst_stage_mask,
                                         VkAccessFlags src_access,
@@ -1633,6 +1648,9 @@ void RendererVulkan::DrawListEnd() {
 }
 
 void RendererVulkan::SwapBuffers() {
+  // Ensure all tasks in the background thread are complete.
+  task_runner_.WaitForCompletion();
+
   vkEndCommandBuffer(frames_[current_frame_].setup_command_buffer);
   vkEndCommandBuffer(frames_[current_frame_].draw_command_buffer);
 
@@ -1646,6 +1664,32 @@ void RendererVulkan::SwapBuffers() {
   BeginFrame();
 }
 
+void RendererVulkan::SetupThreadMain(int preallocate) {
+  if (max_staging_buffer_size_ < staging_buffer_size_ * 4)
+    max_staging_buffer_size_ = staging_buffer_size_ * 4;
+
+  current_staging_buffer_ = 0;
+  staging_buffer_used_ = false;
+
+  for (int i = 0; i < preallocate; i++) {
+    bool err = InsertStagingBuffer();
+    LOG_IF(!err) << "Failed to create staging buffer.";
+  }
+
+  for (;;) {
+    semaphore_.Acquire();
+    if (quit_.load(std::memory_order_relaxed))
+      break;
+
+    task_runner_.SingleConsumerRun();
+  }
+
+  for (int i = 0; i < staging_buffers_.size(); i++) {
+    auto [buffer, allocation] = staging_buffers_[i].buffer;
+    vmaDestroyBuffer(allocator_, buffer, allocation);
+  }
+}
+
 template <typename T>
 bool RendererVulkan::SetUniformInternal(ShaderVulkan* shader,
                                         const std::string& name,

src/engine/renderer/vulkan/renderer_vulkan.h

@@ -1,13 +1,17 @@
 #ifndef RENDERER_VULKAN_H
 #define RENDERER_VULKAN_H
 
+#include <atomic>
 #include <memory>
 #include <string>
+#include <thread>
 #include <unordered_map>
 #include <vector>
 
 #include "vulkan_context.h"
 
+#include "../../../base/semaphore.h"
+#include "../../../base/task_runner.h"
 #include "../../../third_party/vma/vk_mem_alloc.h"
 #include "../render_resource.h"
 #include "../renderer.h"
@@ -128,8 +132,9 @@ class RendererVulkan : public Renderer {
   // be destroyed when the frame is cycled. There are 2 or 3 frames (double or
   // tripple buffering) that are cycled constantly.
   struct Frame {
-    VkCommandPool command_pool = VK_NULL_HANDLE;
+    VkCommandPool setup_command_pool = VK_NULL_HANDLE;
     VkCommandBuffer setup_command_buffer = VK_NULL_HANDLE;
+    VkCommandPool draw_command_pool = VK_NULL_HANDLE;
     VkCommandBuffer draw_command_buffer = VK_NULL_HANDLE;
 
     BufferDeathRow buffers_to_destroy;
@@ -171,6 +176,11 @@ class RendererVulkan : public Renderer {
 
   std::unordered_map<unsigned, RenderResource*> resources_;
 
+  std::thread setup_thread_;
+  base::TaskRunner task_runner_;
+  base::Semaphore semaphore_;
+  std::atomic<bool> quit_{false};
+
 #if defined(__ANDROID__)
   ANativeWindow* window_;
 #elif defined(__linux__)
@@ -182,7 +192,7 @@ class RendererVulkan : public Renderer {
 
   void BeginFrame();
 
-  void Flush();
+  void FlushSetupBuffer();
 
   void FreePendingResources(int frame);
 
@@ -206,7 +216,7 @@ class RendererVulkan : public Renderer {
                       uint32_t usage,
                       VmaMemoryUsage mapping);
   void FreeBuffer(Buffer<VkBuffer> buffer);
-  bool UpdateBuffer(VkBuffer buffer,
+  void UpdateBuffer(VkBuffer buffer,
                     size_t offset,
                     const void* data,
                     size_t data_size);
@@ -228,8 +238,8 @@ class RendererVulkan : public Renderer {
   void FreeTexture(Buffer<VkImage> image,
                    VkImageView image_view,
                    DescSet desc_set);
-  bool UpdateImage(VkImage image, const uint8_t* data, int width, int height);
+  void UpdateImage(VkImage image, const uint8_t* data, int width, int height);
-  void ImageMemoryBarrier(VkImage& image,
+  void ImageMemoryBarrier(VkImage image,
                           VkPipelineStageFlags src_stage_mask,
                           VkPipelineStageFlags dst_stage_mask,
                           VkAccessFlags src_access,
@@ -244,6 +254,8 @@ class RendererVulkan : public Renderer {
 
   void SwapBuffers();
 
+  void SetupThreadMain(int preallocate);
+
   template <typename T>
   bool SetUniformInternal(ShaderVulkan* shader, const std::string& name, T val);
 

src/engine/renderer/vulkan/vulkan_context.cc

@@ -1204,11 +1204,15 @@ bool VulkanContext::UpdateSwapChain(Window* window) {
   return true;
 }
 
-void VulkanContext::AppendCommandBuffer(const VkCommandBuffer& command_buffer) {
+void VulkanContext::AppendCommandBuffer(const VkCommandBuffer& command_buffer,
+                                        bool front) {
+  if (front)
+    command_buffers_.insert(command_buffers_.begin(), command_buffer);
+  else
     command_buffers_.push_back(command_buffer);
 }
 
-void VulkanContext::Flush() {
+void VulkanContext::Flush(bool all) {
   // Ensure everything else pending is executed.
   vkDeviceWaitIdle(device_);
 
@@ -1219,7 +1223,7 @@ void VulkanContext::Flush() {
   submit_info.pWaitDstStageMask = nullptr;
   submit_info.waitSemaphoreCount = 0;
   submit_info.pWaitSemaphores = nullptr;
-  submit_info.commandBufferCount = command_buffers_.size();
+  submit_info.commandBufferCount = command_buffers_.size() - (all ? 0 : 1);
   submit_info.pCommandBuffers = command_buffers_.data();
   submit_info.signalSemaphoreCount = 0;
   submit_info.pSignalSemaphores = nullptr;
@@ -1231,7 +1235,10 @@ void VulkanContext::Flush() {
     return;
   }
 
+  auto draw_command_buffer = command_buffers_.back();
   command_buffers_.clear();
+  if (!all)
+    command_buffers_.push_back(draw_command_buffer);
 
   vkDeviceWaitIdle(device_);
 }

src/engine/renderer/vulkan/vulkan_context.h

@@ -42,9 +42,10 @@ class VulkanContext {
 
   VkExtent2D GetSwapchainExtent() { return window_.swapchain_extent; }
 
-  void AppendCommandBuffer(const VkCommandBuffer& command_buffer);
+  void AppendCommandBuffer(const VkCommandBuffer& command_buffer,
+                           bool front = false);
 
-  void Flush();
+  void Flush(bool all);
 
   bool PrepareBuffers();
   bool SwapBuffers();