parallel-gx1/Source/Core/VideoCommon/Fifo.cpp

// Copyright 2008 Dolphin Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#include "VideoCommon/Fifo.h"

#include <features/features_cpu.h>

#include <cstring>

#include "Common/Assert.h"
#include "Common/BlockingLoop.h"
#include "Common/ChunkFile.h"
#include "Common/Event.h"
#include "Common/FPURoundMode.h"
#include "Common/MemoryUtil.h"
#include "Common/MsgHandler.h"

#include "Core/Config/MainSettings.h"
#include "Core/ConfigManager.h"
#include "Core/CoreTiming.h"
#include "Core/HW/GPFifo.h"
#include "Core/HW/Memmap.h"
#include "Core/Host.h"
#include "Core/System.h"

#include "VideoCommon/AsyncRequests.h"
#include "VideoCommon/CPMemory.h"
#include "VideoCommon/CommandProcessor.h"
#include "VideoCommon/DataReader.h"
#include "VideoCommon/FramebufferManager.h"
#include "VideoCommon/OpcodeDecoding.h"
#include "VideoCommon/VertexLoaderManager.h"
#include "VideoCommon/VertexManagerBase.h"
#include "VideoCommon/VideoBackendBase.h"

namespace Fifo
{
using CommandProcessor::LoadReg;
using CommandProcessor::StoreReg;

static constexpr int GPU_TIME_SLOT_SIZE = 1000;

FifoManager::FifoManager(Core::System& system) : m_system{system}
{
}

FifoManager::~FifoManager() = default;

void FifoManager::RefreshConfig()
{
  m_config_sync_gpu = Config::Get(Config::MAIN_SYNC_GPU);
  m_config_sync_gpu_max_distance = Config::Get(Config::MAIN_SYNC_GPU_MAX_DISTANCE);
  m_config_sync_gpu_min_distance = Config::Get(Config::MAIN_SYNC_GPU_MIN_DISTANCE);
  m_config_sync_gpu_overclock = Config::Get(Config::MAIN_SYNC_GPU_OVERCLOCK);
}

void FifoManager::DoState(PointerWrap& p)
{
  p.DoArray(m_video_buffer, FIFO_SIZE);
  u8* write_ptr = GetWritePtr();
  p.DoPointer(write_ptr, m_video_buffer);
  SetWritePtr(write_ptr);
  p.DoPointer(m_video_buffer_read_ptr, m_video_buffer);
  if (p.IsReadMode() && m_use_deterministic_gpu_thread)
  {
    // We're good and paused, right?
    m_video_buffer_pp_read_ptr = m_video_buffer_read_ptr;
    SetSeenPtr(m_video_buffer_read_ptr);
  }

  p.Do(m_sync_ticks);
  p.Do(m_syncing_suspended);
}

void FifoManager::PauseAndLock()
{
  SyncGPU(SyncGPUReason::Other);
  EmulatorState(false);

  if (!m_system.IsDualCoreMode() || m_use_deterministic_gpu_thread)
    return;

  m_gpu_mainloop.Wait();
}

void FifoManager::RestoreState(const bool was_running)
{
  if (was_running)
    EmulatorState(true);
}

void FifoManager::Init()
{
  if (!m_config_callback_id)
    m_config_callback_id = Config::AddConfigChangedCallback([this] { RefreshConfig(); });
  RefreshConfig();

  // Padded so that SIMD overreads in the vertex loader are safe
  m_video_buffer = static_cast<u8*>(Common::AllocateMemoryPages(FIFO_SIZE + 4));
  ResetVideoBuffer();
  if (m_system.IsDualCoreMode())
    m_gpu_mainloop.Prepare();
  retro_atomic_store_release_int(&m_sync_ticks, 0);
}

void FifoManager::Shutdown()
{
  if (m_gpu_mainloop.IsRunning())
    PanicAlertFmt("FIFO shutting down while active");

  Common::FreeMemoryPages(m_video_buffer, FIFO_SIZE + 4);
  m_video_buffer = nullptr;
  SetWritePtr(nullptr);
  m_video_buffer_pp_read_ptr = nullptr;
  m_video_buffer_read_ptr = nullptr;
  SetSeenPtr(nullptr);
  m_fifo_aux_write_ptr = nullptr;
  m_fifo_aux_read_ptr = nullptr;

  if (m_config_callback_id)
  {
    Config::RemoveConfigChangedCallback(*m_config_callback_id);
    m_config_callback_id = std::nullopt;
  }
}

// May be executed from any thread, even the graphics thread.
// Created to allow for self shutdown.
void FifoManager::ExitGpuLoop()
{
  auto& command_processor = m_system.GetCommandProcessor();
  auto& fifo = command_processor.GetFifo();

  // This should break the wait loop in CPU thread
  StoreReg(&fifo.bFF_GPReadEnable, 0);
  FlushGpu();

  // Terminate GPU thread loop
  m_emu_running_state.Set();
  m_gpu_mainloop.Stop(Common::BlockingLoop::StopMode::NonBlock);
}

// Called by the CPU thread at the end of the frame. It can get there before the frontend
// thread has entered RunGpuLoop(); the loop of the last frame then still counts as stopped
// and this Stop() does nothing, so the request is kept for RunGpuLoop() to act on.
void FifoManager::StopGpuLoop()
{
  retro_atomic_store_release_int(&m_stop_gpu_loop_requested, 1);
  m_gpu_mainloop.Stop(Common::BlockingLoop::StopMode::NonBlock);
}

void FifoManager::EmulatorState(bool running)
{
  m_emu_running_state.Set(running);
  if (running)
    m_gpu_mainloop.Wakeup();
}

/* Any thread: the loop returns soon, its field not ended, so its thread
 * can look at what it was asked; kept until a run takes it, when no run
 * is going. */
void FifoManager::InterruptGpuLoop()
{
  retro_atomic_store_release_int(&m_gpu_loop_interrupt, 1);
  m_gpu_mainloop.Stop(Common::BlockingLoop::StopMode::NonBlock);
}

void FifoManager::SyncGPU(SyncGPUReason reason, bool may_move_read_ptr)
{
  if (m_use_deterministic_gpu_thread)
  {
    m_gpu_mainloop.Wait();
    if (!m_gpu_mainloop.IsRunning())
      return;

    // Opportunistically reset FIFOs so we don't wrap around.
    if (may_move_read_ptr && m_fifo_aux_write_ptr != m_fifo_aux_read_ptr)
    {
      PanicAlertFmt("Aux FIFO not synced ({}, {})", fmt::ptr(m_fifo_aux_write_ptr),
                    fmt::ptr(m_fifo_aux_read_ptr));
    }

    memmove(m_fifo_aux_data, m_fifo_aux_read_ptr, m_fifo_aux_write_ptr - m_fifo_aux_read_ptr);
    m_fifo_aux_write_ptr -= (m_fifo_aux_read_ptr - m_fifo_aux_data);
    m_fifo_aux_read_ptr = m_fifo_aux_data;

    if (may_move_read_ptr)
    {
      u8* write_ptr = GetWritePtr();

      // what's left over in the buffer
      size_t size = write_ptr - m_video_buffer_pp_read_ptr;

      memmove(m_video_buffer, m_video_buffer_pp_read_ptr, size);
      // This change always decreases the pointers.  We write seen_ptr
      // after write_ptr here, and read it before in RunGpuLoop, so
      // 'write_ptr > seen_ptr' there cannot become spuriously true.
      write_ptr = m_video_buffer + size;
      SetWritePtr(write_ptr);
      m_video_buffer_pp_read_ptr = m_video_buffer;
      m_video_buffer_read_ptr = m_video_buffer;
      SetSeenPtr(write_ptr);
    }
  }
}

void FifoManager::PushFifoAuxBuffer(const void* ptr, size_t size)
{
  if (size > (size_t)(m_fifo_aux_data + FIFO_SIZE - m_fifo_aux_write_ptr))
  {
    SyncGPU(SyncGPUReason::AuxSpace, /* may_move_read_ptr */ false);
    if (!m_gpu_mainloop.IsRunning())
    {
      // GPU is shutting down
      return;
    }
    if (size > (size_t)(m_fifo_aux_data + FIFO_SIZE - m_fifo_aux_write_ptr))
    {
      // That will sync us up to the last 32 bytes, so this short region
      // of FIFO would have to point to a 2MB display list or something.
      PanicAlertFmt("Absurdly large aux buffer");
      return;
    }
  }
  memcpy(m_fifo_aux_write_ptr, ptr, size);
  m_fifo_aux_write_ptr += size;
}

void* FifoManager::PopFifoAuxBuffer(size_t size)
{
  void* ret = m_fifo_aux_read_ptr;
  m_fifo_aux_read_ptr += size;
  return ret;
}

// Description: RunGpuLoop() sends data through this function.
void FifoManager::ReadDataFromFifo(u32 read_ptr)
{
  u8* write_ptr = GetWritePtr();
  if (GPFifo::GATHER_PIPE_SIZE > static_cast<size_t>(m_video_buffer + FIFO_SIZE - write_ptr))
  {
    const size_t existing_len = write_ptr - m_video_buffer_read_ptr;
    if (GPFifo::GATHER_PIPE_SIZE > static_cast<size_t>(FIFO_SIZE - existing_len))
    {
      PanicAlertFmt("FIFO out of bounds (existing {} + new {} > {})", existing_len,
                    GPFifo::GATHER_PIPE_SIZE, FIFO_SIZE);
      return;
    }
    memmove(m_video_buffer, m_video_buffer_read_ptr, existing_len);
    write_ptr = m_video_buffer + existing_len;
    m_video_buffer_read_ptr = m_video_buffer;
  }
  // Copy new video instructions to m_video_buffer for future use in rendering the new picture
  auto& memory = m_system.GetMemory();
  memory.CopyFromEmu(write_ptr, read_ptr, GPFifo::GATHER_PIPE_SIZE);
  SetWritePtr(write_ptr + GPFifo::GATHER_PIPE_SIZE);
}

// The deterministic_gpu_thread version.
void FifoManager::ReadDataFromFifoOnCPU(u32 read_ptr)
{
  u8* write_ptr = GetWritePtr();
  if (GPFifo::GATHER_PIPE_SIZE > static_cast<size_t>(m_video_buffer + FIFO_SIZE - write_ptr))
  {
    // We can't wrap around while the GPU is working on the data.
    // This should be very rare due to the reset in SyncGPU.
    SyncGPU(SyncGPUReason::Wraparound);
    if (!m_gpu_mainloop.IsRunning())
    {
      // GPU is shutting down, so the next asserts may fail
      return;
    }

    if (m_video_buffer_pp_read_ptr != m_video_buffer_read_ptr)
    {
      PanicAlertFmt("Desynced read pointers");
      return;
    }
    write_ptr = GetWritePtr();
    const size_t existing_len = write_ptr - m_video_buffer_pp_read_ptr;
    if (GPFifo::GATHER_PIPE_SIZE > static_cast<size_t>(FIFO_SIZE - existing_len))
    {
      PanicAlertFmt("FIFO out of bounds (existing {} + new {} > {})", existing_len,
                    GPFifo::GATHER_PIPE_SIZE, FIFO_SIZE);
      return;
    }
  }
  auto& memory = m_system.GetMemory();
  memory.CopyFromEmu(write_ptr, read_ptr, GPFifo::GATHER_PIPE_SIZE);
  m_video_buffer_pp_read_ptr = OpcodeDecoder::RunFifo<true>(
      DataReader(m_video_buffer_pp_read_ptr, write_ptr + GPFifo::GATHER_PIPE_SIZE), nullptr);
  // This would have to be locked if the GPU thread didn't spin.
  SetWritePtr(write_ptr + GPFifo::GATHER_PIPE_SIZE);
}

void FifoManager::ResetVideoBuffer()
{
  m_video_buffer_read_ptr = m_video_buffer;
  SetWritePtr(m_video_buffer);
  SetSeenPtr(m_video_buffer);
  m_video_buffer_pp_read_ptr = m_video_buffer;
  m_fifo_aux_write_ptr = m_fifo_aux_data;
  m_fifo_aux_read_ptr = m_fifo_aux_data;
}

// Description: Main FIFO update loop
// Purpose: Keep the Core HW updated about the CPU-GPU distance
int FifoManager::RunGpuLoop(s64 wedge_us)
{
  bool interrupted = false;

  m_gpu_mainloop.SetIdleTimeout(wedge_us);
  m_gpu_mainloop.Run(
      [&] {
        if (retro_atomic_load_acquire_int(&m_stop_gpu_loop_requested))
          m_gpu_mainloop.Stop(Common::BlockingLoop::StopMode::NonBlock);
        else if (retro_atomic_exchange_int(&m_gpu_loop_interrupt, 0))
        {
          interrupted = true;
          m_gpu_mainloop.Stop(Common::BlockingLoop::StopMode::NonBlock);
        }

        // Run events from the CPU thread.
        AsyncRequests::GetInstance()->PullEvents();

        // Do nothing while paused
        if (!m_emu_running_state.IsSet())
          return;

        if (m_use_deterministic_gpu_thread)
        {
          // All the fifo/CP stuff is on the CPU.  We just need to run the opcode decoder.
          u8* seen_ptr = GetSeenPtr();
          u8* write_ptr = GetWritePtr();
          // See comment in SyncGPU
          if (write_ptr > seen_ptr)
          {
            m_video_buffer_read_ptr =
                OpcodeDecoder::RunFifo(DataReader(m_video_buffer_read_ptr, write_ptr), nullptr);
            SetSeenPtr(write_ptr);
          }
        }
        else
        {
          auto& command_processor = m_system.GetCommandProcessor();
          auto& fifo = command_processor.GetFifo();
          command_processor.SetCPStatusFromGPU();

          // check if we are able to run this buffer
          while (!command_processor.IsInterruptWaiting() && LoadReg(&fifo.bFF_GPReadEnable) &&
                 LoadReg(&fifo.CPReadWriteDistance) && !AtBreakpoint(m_system))
          {
            if (m_config_sync_gpu &&
                retro_atomic_load_seq_cst_int(&m_sync_ticks) < m_config_sync_gpu_min_distance)
              break;

            u32 cyclesExecuted = 0;
            u32 readPtr = LoadReg(&fifo.CPReadPointer);
            ReadDataFromFifo(readPtr);

            if (readPtr == LoadReg(&fifo.CPEnd))
              readPtr = LoadReg(&fifo.CPBase);
            else
              readPtr += GPFifo::GATHER_PIPE_SIZE;

            const s32 distance =
                static_cast<s32>(LoadReg(&fifo.CPReadWriteDistance)) - GPFifo::GATHER_PIPE_SIZE;
            ASSERT_MSG(COMMANDPROCESSOR, distance >= 0,
                       "Negative fifo.CPReadWriteDistance = {} in FIFO Loop !\nThat can produce "
                       "instability in the game. Please report it.",
                       distance);

            u8* write_ptr = GetWritePtr();
            m_video_buffer_read_ptr = OpcodeDecoder::RunFifo(
                DataReader(m_video_buffer_read_ptr, write_ptr), &cyclesExecuted);

            StoreReg(&fifo.CPReadPointer, readPtr);
            retro_atomic_fetch_add_seq_cst_int(&fifo.CPReadWriteDistance,
                                               -static_cast<int>(GPFifo::GATHER_PIPE_SIZE));
            if ((write_ptr - m_video_buffer_read_ptr) == 0)
            {
              StoreReg(&fifo.SafeCPReadPointer, LoadReg(&fifo.CPReadPointer));
            }

            command_processor.SetCPStatusFromGPU();

            if (m_config_sync_gpu)
            {
              cyclesExecuted = (int)(cyclesExecuted / m_config_sync_gpu_overclock);
              int old = retro_atomic_fetch_add_seq_cst_int(&m_sync_ticks,
                                                           -static_cast<int>(cyclesExecuted));
              if (old >= m_config_sync_gpu_max_distance &&
                  old - (int)cyclesExecuted < m_config_sync_gpu_max_distance)
              {
                m_sync_wakeup_event.Set();
              }
            }

            // This call is pretty important in DualCore mode and must be called in the FIFO Loop.
            // If we don't, s_swapRequested or s_efbAccessRequested won't be set to false
            // leading the CPU thread to wait in Video_OutputXFB or Video_AccessEFB thus slowing
            // things down.
            AsyncRequests::GetInstance()->PullEvents();
          }

          // fast skip remaining GPU time if fifo is empty
          if (retro_atomic_load_seq_cst_int(&m_sync_ticks) > 0)
          {
            int old = retro_atomic_exchange_int(&m_sync_ticks, 0);
            if (old >= m_config_sync_gpu_max_distance)
              m_sync_wakeup_event.Set();
          }

          // The fifo is empty and it's unlikely we will get any more work in the near future.
          // Make sure VertexManager finishes drawing any primitives it has stored in it's buffer.
          g_vertex_manager->Flush();
          g_framebuffer_manager->RefreshPeekCache();
        }
      });
  const bool wedged = m_gpu_mainloop.IdleTimedOut();
  m_gpu_mainloop.SetIdleTimeout(0);
  /* a field's end that came while the loop was interrupted or idle too
   * long is kept for the next run, which then finishes that field */
  if (interrupted)
    return GPU_LOOP_INTERRUPTED;
  if (wedged)
    return GPU_LOOP_WEDGED;
  retro_atomic_store_release_int(&m_stop_gpu_loop_requested, 0);
  return GPU_LOOP_FIELD;
}

void FifoManager::FlushGpu()
{
  if (!m_system.IsDualCoreMode() || m_use_deterministic_gpu_thread)
    return;

  m_gpu_mainloop.Wait();
}

bool AtBreakpoint(Core::System& system)
{
  auto& command_processor = system.GetCommandProcessor();
  const auto& fifo = command_processor.GetFifo();
  return LoadReg(&fifo.bFF_BPEnable) &&
         (LoadReg(&fifo.CPReadPointer) == LoadReg(&fifo.CPBreakpoint));
}

void FifoManager::RunGpu()
{
  const bool is_dual_core = m_system.IsDualCoreMode();

  // wake up GPU thread
  if (is_dual_core && !m_use_deterministic_gpu_thread)
  {
    m_gpu_mainloop.Wakeup();
  }

  // if the sync GPU callback is suspended, wake it up.
  if (!is_dual_core || m_use_deterministic_gpu_thread || m_config_sync_gpu)
  {
    if (m_syncing_suspended)
    {
      m_syncing_suspended = false;
      m_system.GetCoreTiming().ScheduleEvent(GPU_TIME_SLOT_SIZE, m_event_sync_gpu,
                                             GPU_TIME_SLOT_SIZE);
    }
  }
}

int FifoManager::RunGpuOnCpu(int ticks)
{
  auto& command_processor = m_system.GetCommandProcessor();
  auto& fifo = command_processor.GetFifo();
  bool reset_simd_state = false;
  int available_ticks =
      int(ticks * m_config_sync_gpu_overclock) + retro_atomic_load_seq_cst_int(&m_sync_ticks);
  while (LoadReg(&fifo.bFF_GPReadEnable) && LoadReg(&fifo.CPReadWriteDistance) &&
         !AtBreakpoint(m_system) && available_ticks >= 0)
  {
    if (m_use_deterministic_gpu_thread)
    {
      ReadDataFromFifoOnCPU(LoadReg(&fifo.CPReadPointer));
      m_gpu_mainloop.Wakeup();
    }
    else
    {
      if (!reset_simd_state)
      {
        Common::FPU::SaveSIMDState();
        Common::FPU::LoadDefaultSIMDState();
        reset_simd_state = true;
      }
      ReadDataFromFifo(LoadReg(&fifo.CPReadPointer));
      u32 cycles = 0;
      m_video_buffer_read_ptr =
          OpcodeDecoder::RunFifo(DataReader(m_video_buffer_read_ptr, GetWritePtr()), &cycles);
      available_ticks -= cycles;
    }

    if (LoadReg(&fifo.CPReadPointer) == LoadReg(&fifo.CPEnd))
    {
      StoreReg(&fifo.CPReadPointer, LoadReg(&fifo.CPBase));
    }
    else
    {
      retro_atomic_fetch_add_int(&fifo.CPReadPointer, GPFifo::GATHER_PIPE_SIZE);
    }

    retro_atomic_fetch_sub_int(&fifo.CPReadWriteDistance, GPFifo::GATHER_PIPE_SIZE);
  }

  command_processor.SetCPStatusFromGPU();

  if (reset_simd_state)
  {
    Common::FPU::LoadSIMDState();
  }

  // Discard all available ticks as there is nothing to do any more.
  retro_atomic_store_release_int(&m_sync_ticks, std::min(available_ticks, 0));

  // If the GPU is idle, drop the handler.
  if (available_ticks >= 0)
    return -1;

  // Always wait at least for GPU_TIME_SLOT_SIZE cycles.
  return -available_ticks + GPU_TIME_SLOT_SIZE;
}

void FifoManager::UpdateWantDeterminism(bool want)
{
  // We are paused (or not running at all yet), so
  // it should be safe to change this.
  bool gpu_thread = false;
  switch (Config::GetGPUDeterminismMode())
  {
  case Config::GPUDeterminismMode::Auto:
    gpu_thread = want;
    break;
  case Config::GPUDeterminismMode::Disabled:
    gpu_thread = false;
    break;
  case Config::GPUDeterminismMode::FakeCompletion:
    gpu_thread = true;
    break;
  }

  gpu_thread = gpu_thread && m_system.IsDualCoreMode();

  if (m_use_deterministic_gpu_thread != gpu_thread)
  {
    m_use_deterministic_gpu_thread = gpu_thread;
    if (gpu_thread)
    {
      // These haven't been updated in non-deterministic mode.
      m_video_buffer_pp_read_ptr = m_video_buffer_read_ptr;
      SetSeenPtr(m_video_buffer_read_ptr);
      CopyPreprocessCPStateFromMain();
      VertexLoaderManager::MarkAllDirty();
    }
  }
}

/* This function checks the emulated CPU - GPU distance and may wake up the GPU,
 * or block the CPU if required. It should be called by the CPU thread regularly.
 * @ticks The gone emulated CPU time.
 * @return A good time to call WaitForGpuThread() next.
 */
int FifoManager::WaitForGpuThread(int ticks)
{
  int old = retro_atomic_fetch_add_seq_cst_int(&m_sync_ticks, ticks);
  int now = old + ticks;

  // GPU is idle, so stop polling.
  if (old >= 0 && m_gpu_mainloop.IsDone())
    return -1;

  // Wakeup GPU
  if (old < m_config_sync_gpu_min_distance && now >= m_config_sync_gpu_min_distance)
    RunGpu();

  // If the GPU is still sleeping, wait for a longer time
  if (now < m_config_sync_gpu_min_distance)
    return GPU_TIME_SLOT_SIZE + m_config_sync_gpu_min_distance - now;

  // Wait for GPU
  if (now >= m_config_sync_gpu_max_distance)
    m_sync_wakeup_event.Wait();

  return GPU_TIME_SLOT_SIZE;
}

void FifoManager::SyncGPUCallback(Core::System& system, u64 ticks, s64 cyclesLate)
{
  ticks += cyclesLate;
  int next = -1;

  auto& fifo = system.GetFifo();
  if (!system.IsDualCoreMode() || fifo.m_use_deterministic_gpu_thread)
  {
    next = fifo.RunGpuOnCpu(int(ticks));
  }
  else if (fifo.m_config_sync_gpu)
  {
    next = fifo.WaitForGpuThread(int(ticks));
  }

  fifo.m_syncing_suspended = next < 0;
  if (!fifo.m_syncing_suspended)
    system.GetCoreTiming().ScheduleEvent(next, fifo.m_event_sync_gpu, next);
}

void FifoManager::SyncGPUForRegisterAccess()
{
  SyncGPU(SyncGPUReason::Other);

  if (!m_system.IsDualCoreMode() || m_use_deterministic_gpu_thread)
    RunGpuOnCpu(GPU_TIME_SLOT_SIZE);
  else if (m_config_sync_gpu)
    WaitForGpuThread(GPU_TIME_SLOT_SIZE);
}

// Initialize GPU - CPU thread syncing, this gives us a deterministic way to start the GPU thread.
void FifoManager::Prepare()
{
  m_event_sync_gpu = m_system.GetCoreTiming().RegisterEvent("SyncGPUCallback", SyncGPUCallback);
  m_syncing_suspended = true;
}
}  // namespace Fifo