parallel-gx1/Source/Core/VideoBackends/Vulkan/CommandBufferManager.h

// Copyright 2016 Dolphin Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include <retro_atomic.h>

#include <Common/WorkQueueThread.h>

#include "VideoBackends/Vulkan/Constants.h"

namespace Vulkan
{
class CommandBufferManager
{
public:
  explicit CommandBufferManager(bool use_threaded_submission);
  ~CommandBufferManager();

  bool Initialize();

  // These command buffers are allocated per-frame. They are valid until the command buffer
  // is submitted, after that you should call these functions again.
  VkCommandBuffer GetCurrentInitCommandBuffer()
  {
    CmdBufferResources& cmd_buffer_resources = GetCurrentCmdBufferResources();
    cmd_buffer_resources.init_command_buffer_used = true;
    return cmd_buffer_resources.command_buffers[0];
  }
  VkCommandBuffer GetCurrentCommandBuffer() const
  {
    const CmdBufferResources& cmd_buffer_resources = m_command_buffers[m_current_cmd_buffer];
    return cmd_buffer_resources.command_buffers[1];
  }
  // Allocates a descriptors set from the pool reserved for the current frame.
  VkDescriptorSet AllocateDescriptorSet(VkDescriptorSetLayout set_layout);

  // Fence "counters" are used to track which commands have been completed by the GPU.
  // If the last completed fence counter is greater or equal to N, it means that the work
  // associated counter N has been completed by the GPU. The value of N to associate with
  // commands can be retrieved by calling GetCurrentFenceCounter().
  u64 GetCompletedFenceCounter() const { return m_completed_fence_counter; }

  // Gets the fence that will be signaled when the currently executing command buffer is
  // queued and executed. Do not wait for this fence before the buffer is executed.
  u64 GetCurrentFenceCounter() const
  {
    auto& resources = m_command_buffers[m_current_cmd_buffer];
    return resources.fence_counter;
  }

  // Ensure that the worker thread has submitted any previous command buffers and is idle.
  void WaitForWorkerThreadIdle();

  // Wait for a fence to be completed.
  // Also invokes callbacks for completion.
  void WaitForFenceCounter(u64 fence_counter);
  // Whether the work of fence_counter is done; never waits.
  bool PollFenceCounter(u64 fence_counter);

  void SubmitCommandBuffer(bool submit_on_worker_thread, bool wait_for_completion,
                           bool advance_to_next_frame = false);

  /* Called with the current command buffer before it ends: what the
   * frontend reads at any time, from another thread, is put back in the
   * layout it reads it in, so every submission leaves it so. */
  void SetBeforeSubmit(void (*callback)(void* opaque, VkCommandBuffer command_buffer),
                       void* opaque)
  {
    m_before_submit = callback;
    m_before_submit_opaque = opaque;
  }

  // Schedule a vulkan resource for destruction later on. This will occur when the command buffer
  // is next re-used, and the GPU has finished working with the specified resource.
  void DeferBufferViewDestruction(VkBufferView object);
  void DeferBufferDestruction(VkBuffer buffer, const vk_heap_alloc_t& alloc);
  void DeferFramebufferDestruction(VkFramebuffer object);
  void DeferImageDestruction(VkImage object, const vk_heap_alloc_t& alloc);
  void DeferImageViewDestruction(VkImageView object);

private:
  bool CreateCommandBuffers();
  void DestroyCommandBuffers();

  bool CreateSubmitThread();

  void WaitForCommandBufferCompletion(u32 command_buffer_index);
  void SubmitCommandBuffer(u32 command_buffer_index);
  void BeginCommandBuffer();

  VkDescriptorPool CreateDescriptorPool(u32 descriptor_sizes);

  const u32 DESCRIPTOR_SETS_PER_POOL = 1024;

  /* An object destroyed once the GPU is done with the command buffer it was last used in. */
  struct DeferredDestruction
  {
    enum Kind
    {
      BUFFER_VIEW,
      BUFFER,
      FRAMEBUFFER,
      IMAGE,
      IMAGE_VIEW
    } kind;
    union
    {
      VkBufferView buffer_view;
      VkBuffer buffer;
      VkFramebuffer framebuffer;
      VkImage image;
      VkImageView image_view;
    };
    vk_heap_alloc_t alloc; /* BUFFER and IMAGE */
  };
  static void Destroy(DeferredDestruction* d);
  void Defer(const DeferredDestruction& d);

  struct CmdBufferResources
  {
    // [0] - Init (upload) command buffer, [1] - draw command buffer
    VkCommandPool command_pool = VK_NULL_HANDLE;
    VkCommandBuffer command_buffers[2] = {};
    VkFence fence = VK_NULL_HANDLE;
    u64 fence_counter = 0;
    bool init_command_buffer_used = false;
    retro_atomic_int_t waiting_for_submit{};
    u32 frame_index = 0;

    /* kept after each clear: grows to what a command buffer frees, then no more */
    DeferredDestruction* cleanup = nullptr;
    u32 num_cleanup = 0;
    u32 cleanup_capacity = 0;
  };

  struct FrameResources
  {
    VkDescriptorPool* descriptor_pools = nullptr;
    u32 num_descriptor_pools = 0;
    u32 descriptor_pools_capacity = 0;
    u32 current_descriptor_pool_index = 0;
  };

  FrameResources& GetCurrentFrameResources() { return m_frame_resources[m_current_frame]; }

  CmdBufferResources& GetCurrentCmdBufferResources()
  {
    return m_command_buffers[m_current_cmd_buffer];
  }

  u64 m_next_fence_counter = 1;
  u64 m_completed_fence_counter = 0;

  FrameResources m_frame_resources[NUM_FRAMES_IN_FLIGHT];
  CmdBufferResources m_command_buffers[NUM_COMMAND_BUFFERS];
  u32 m_current_frame = 0;
  u32 m_current_cmd_buffer = 0;

  // Threaded command buffer execution
  struct PendingCommandBufferSubmit
  {
    u32 command_buffer_index;
  };
  Common::WorkQueueThreadSP<PendingCommandBufferSubmit> m_submit_thread;
  bool m_use_threaded_submission = false;
  u32 m_descriptor_set_count = DESCRIPTOR_SETS_PER_POOL;
  void (*m_before_submit)(void* opaque, VkCommandBuffer command_buffer) = nullptr;
  void* m_before_submit_opaque = nullptr;
};

extern CommandBufferManager* g_command_buffer_mgr;

// Whether the GPU did the work of fence_counter; never waits.
bool FenceCounterDone(u64 fence_counter);

}  // namespace Vulkan