From d78aef2b77122461c5abbe1e4193b1d92506acaa Mon Sep 17 00:00:00 2001
From: Souryo <mesenemu@gmail.com>
Date: Wed, 2 Aug 2017 18:43:38 -0400
Subject: [PATCH] Debugger: Fixed crash on games that never call RTI

---
 Core/Debugger.cpp | 19 +++++++++++++++++--
 Core/Debugger.h   |  5 ++---
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/Core/Debugger.cpp b/Core/Debugger.cpp
index 6182dce4..b4e3db57 100644
--- a/Core/Debugger.cpp
+++ b/Core/Debugger.cpp
@@ -284,6 +284,17 @@ int32_t Debugger::EvaluateExpression(string expression, EvalResultType &resultTy
 	return _watchExpEval.Evaluate(expression, state, resultType);
 }
 
+void Debugger::RemoveExcessCallstackEntries()
+{
+	while(_callstackRelative.size() >= 1022) {
+		//Ensure callstack stays below 512 entries - some games never call RTI, causing an infinite stack
+		_callstackRelative.pop_front();
+		_callstackRelative.pop_front();
+		_callstackAbsolute.pop_front();
+		_callstackAbsolute.pop_front();
+	}
+}
+
 void Debugger::UpdateCallstack(uint32_t addr)
 {
 	_hideTopOfCallstack = false;
@@ -295,8 +306,10 @@ void Debugger::UpdateCallstack(uint32_t addr)
 		_callstackAbsolute.pop_back();
 
 		_profiler->UnstackFunction();
-	} else if(_lastInstruction == 0x20 && _callstackRelative.size() < 1022) {
+	} else if(_lastInstruction == 0x20) {
 		//JSR
+		RemoveExcessCallstackEntries();
+
 		uint16_t targetAddr = _memoryManager->DebugRead(addr + 1) | (_memoryManager->DebugRead(addr + 2) << 8);
 		_callstackRelative.push_back(addr);
 		_callstackRelative.push_back(targetAddr);
@@ -312,6 +325,8 @@ void Debugger::UpdateCallstack(uint32_t addr)
 
 void Debugger::PrivateProcessInterrupt(uint16_t cpuAddr, uint16_t destCpuAddr, bool forNmi)
 {
+	RemoveExcessCallstackEntries();
+
 	_callstackRelative.push_back(cpuAddr | (forNmi ? 0x40000 : 0x20000));
 	_callstackRelative.push_back(destCpuAddr);
 
@@ -739,7 +754,7 @@ void Debugger::ProcessVramOperation(MemoryOperationType type, uint16_t addr, uin
 
 void Debugger::GetCallstack(int32_t* callstackAbsolute, int32_t* callstackRelative)
 {
-	int callstackSize = (int)_callstackRelative.size() - (_hideTopOfCallstack ? 2 : 0);
+	int callstackSize = std::min(1022, (int)_callstackRelative.size() - (_hideTopOfCallstack ? 2 : 0));
 	for(int i = 0; i < callstackSize; i++) {
 		callstackAbsolute[i] = _callstackAbsolute[i];
 		
diff --git a/Core/Debugger.h b/Core/Debugger.h
index 9ea6b707..61a4beec 100644
--- a/Core/Debugger.h
+++ b/Core/Debugger.h
@@ -119,6 +119,8 @@ private:
 	void ProcessStepConditions(uint32_t addr);
 	bool SleepUntilResume();
 
+	void RemoveExcessCallstackEntries();
+
 public:
 	Debugger(shared_ptr<Console> console, shared_ptr<CPU> cpu, shared_ptr<PPU> ppu, shared_ptr<MemoryManager> memoryManager, shared_ptr<BaseMapper> mapper);
 	~Debugger();
@@ -165,9 +167,6 @@ public:
 	int32_t GetAbsoluteAddress(uint32_t addr);	
 	void GetAbsoluteAddressAndType(uint32_t relativeAddr, AddressTypeInfo* info);
 
-	void StartTraceLogger(TraceLoggerOptions options);
-	void StopTraceLogger();
-
 	shared_ptr<Profiler> GetProfiler();
 	shared_ptr<Assembler> GetAssembler();
 	shared_ptr<TraceLogger> GetTraceLogger();