Tests: General improvments

-Give the number of mismatching frames
-Allow debugger to break when a frame is different
-Fix mismatches between GUI & TestHelper environments (GUI always has four score adapter enabled)

Core/AutoRomTest.cpp

@@ -4,6 +4,7 @@
 #include "Console.h"
 #include "EmulationSettings.h"
 #include "MessageManager.h"
+#include "Debugger.h"
 #include "../Utilities/FolderUtilities.h"
 #include "../Utilities/md5.h"
 #include "../Utilities/ZipWriter.h"
@@ -58,10 +59,11 @@ void AutoRomTest::ValidateFrame(uint16_t* ppuFrameBuffer)
 	_currentCount--;
 
 	if(memcmp(_screenshotHashes.front(), md5Hash, 16) != 0) {
-		_testResult = false;
-		_runningTest = false;
-		_signal.Signal();
-	} else if (_currentCount == 0 && _repetitionCount.empty()) {
+		_badFrameCount++;
+		Debugger::BreakIfDebugging();
+	} 
+	
+	if (_currentCount == 0 && _repetitionCount.empty()) {
 		//End of test
 		_runningTest = false;
 		_signal.Signal();
@@ -100,7 +102,7 @@ void AutoRomTest::Reset()
 
 	_runningTest = false;
 	_recording = false;
-	_testResult = true;
+	_badFrameCount = 0;
 	_recordingFromMovie = false;
 }
 
@@ -177,7 +179,7 @@ void AutoRomTest::RecordFromTest(string newTestFilename, string existingTestFile
 	}
 }
 
-bool AutoRomTest::Run(string filename)
+int AutoRomTest::Run(string filename)
 {
 	ZipReader zipReader;
 	zipReader.LoadZipArchive(filename);
@@ -225,18 +227,18 @@ bool AutoRomTest::Run(string filename)
 		Console::Resume();
 
 		_signal.Wait();
+		_runningTest = false;
 
 		Console::GetInstance()->Stop();
 
-		_runningTest = false;
 
 		EmulationSettings::SetEmulationSpeed(100);
 		EmulationSettings::SetAudioState(true);
 
-		return _testResult;
+		return _badFrameCount;
 	}
 
-	return false;
+	return -1;
 }
 
 void AutoRomTest::Stop()

Core/AutoRomTest.h

@@ -10,7 +10,7 @@ class AutoRomTest : public INotificationListener
 private:
 	bool _recording;
 	bool _runningTest;
-	bool _testResult;
+	int _badFrameCount;
 	bool _recordingFromMovie;
 
 	uint8_t _previousHash[16];
@@ -42,6 +42,6 @@ public:
 	void Record(string filename, bool reset);
 	void RecordFromMovie(string testFilename, string movieFilename);
 	void RecordFromTest(string newTestFilename, string existingTestFilename);
-	bool Run(string filename);
+	int Run(string filename);
 	void Stop();
 };

Core/ControlManager.cpp

@@ -119,7 +119,9 @@ uint8_t ControlManager::GetControllerState(uint8_t controllerID)
 		state = Movie::Instance->GetState(controllerID);
 	} else {
 		if(controlDevice) {
-			_keyManager->RefreshState();
+			if(_keyManager) {
+				_keyManager->RefreshState();
+			}
 			state = controlDevice->GetButtonState().ToByte();
 		} else {
 			state = 0x00;
@@ -140,7 +142,8 @@ uint8_t ControlManager::GetControllerState(uint8_t controllerID)
 
 bool ControlManager::HasFourScoreAdapter()
 {
-	return ControlManager::ControlDevices[2] != nullptr || ControlManager::ControlDevices[3] != nullptr;
+	//When a movie is playing, always assume 4 controllers are plugged in (TODO: Change this so movies know how many controllers were plugged when recording)
+	return ControlManager::ControlDevices[2] != nullptr || ControlManager::ControlDevices[3] != nullptr || Movie::Playing();
 }
 
 void ControlManager::RefreshStateBuffer(uint8_t port)

Core/Debugger.cpp

@@ -49,6 +49,13 @@ Debugger::~Debugger()
 	Console::Resume();
 }
 
+void Debugger::BreakIfDebugging()
+{
+	if(Debugger::Instance != nullptr) {
+		Debugger::Instance->Step(1);
+	}
+}
+
 bool Debugger::LoadCdlFile(string cdlFilepath)
 {
 	if(_codeDataLogger->LoadCdlFile(cdlFilepath)) {

Core/Debugger.h

@@ -114,4 +114,6 @@ public:
 	
 	static void ProcessRamOperation(MemoryOperationType type, uint16_t &addr);
 	static void ProcessVramOperation(MemoryOperationType type, uint16_t addr);
+
+	static void BreakIfDebugging();
 };

GUI.NET/Forms/frmMain.cs

@@ -537,35 +537,37 @@ namespace Mesen.GUI.Forms
 			if(ofd.ShowDialog() == System.Windows.Forms.DialogResult.OK) {
 				List<string> passedTests = new List<string>();
 				List<string> failedTests = new List<string>();
+				List<int> failedFrameCount = new List<int>();
 
 				this.menuStrip.Enabled = false;
 
 				Task.Run(() => {
 					foreach(string filename in ofd.FileNames) {
-						bool result = InteropEmu.RomTestRun(filename);
+						int result = InteropEmu.RomTestRun(filename);
 
-						if(result) {
+						if(result == 0) {
 							passedTests.Add(Path.GetFileNameWithoutExtension(filename));
 						} else {
 							failedTests.Add(Path.GetFileNameWithoutExtension(filename));
+							failedFrameCount.Add(result);
 						}
 					}
 
 					this.BeginInvoke((MethodInvoker)(() => {
 						if(failedTests.Count == 0) {
 							MessageBox.Show("All tests passed.", "", MessageBoxButtons.OK, MessageBoxIcon.Information);
-						} else if(passedTests.Count == 0) {
-							MessageBox.Show("All tests failed.", "", MessageBoxButtons.OK, MessageBoxIcon.Error);
 						} else {
 							StringBuilder message = new StringBuilder();
-							message.AppendLine("Passed tests:");
-							foreach(string test in passedTests) {
-								message.AppendLine("  -" + test);
+							if(passedTests.Count > 0) {
+								message.AppendLine("Passed tests:");
+								foreach(string test in passedTests) {
+									message.AppendLine("  -" + test);
+								}
+								message.AppendLine("");
 							}
-							message.AppendLine("");
 							message.AppendLine("Failed tests:");
-							foreach(string test in failedTests) {
-								message.AppendLine("  -" + test);
+							for(int i = 0, len = failedTests.Count; i < len; i++) {
+								message.AppendLine("  -" + failedTests[i] + " (" + failedFrameCount[i] + ")");
 							}
 							MessageBox.Show(message.ToString(), "", MessageBoxButtons.OK, MessageBoxIcon.Error);
 						}

GUI.NET/InteropEmu.cs

@@ -52,7 +52,7 @@ namespace Mesen.GUI
 		[DllImport(DLLPath)] public static extern bool MoviePlaying();
 		[DllImport(DLLPath)] public static extern bool MovieRecording();
 
-		[DllImport(DLLPath)] public static extern bool RomTestRun([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef=typeof(UTF8Marshaler))]string filename);
+		[DllImport(DLLPath)] public static extern Int32 RomTestRun([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef=typeof(UTF8Marshaler))]string filename);
 		[DllImport(DLLPath)] public static extern void RomTestRecord([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef=typeof(UTF8Marshaler))]string filename, [MarshalAs(UnmanagedType.I1)]bool reset);
 		[DllImport(DLLPath)] public static extern void RomTestRecordFromMovie([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef=typeof(UTF8Marshaler))]string testFilename, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef=typeof(UTF8Marshaler))]string movieFilename);
 		[DllImport(DLLPath)] public static extern void RomTestRecordFromTest([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef=typeof(UTF8Marshaler))]string newTestFilename, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef=typeof(UTF8Marshaler))]string existingTestFilename);

InteropDLL/ConsoleWrapper.cpp

@@ -42,7 +42,7 @@ namespace InteropEmu {
 	};
 
 	extern "C" {
-		DllExport void __stdcall InitializeEmu(char* homeFolder, HWND windowHandle, HWND dxViewerHandle)
+		DllExport void __stdcall InitializeEmu(const char* homeFolder, HWND windowHandle, HWND dxViewerHandle)
 		{
 			FolderUtilities::SetHomeFolder(homeFolder);
 

PGOHelper/PGOHelper.cpp

@@ -5,7 +5,7 @@
 
 extern "C" {
 	void __stdcall InitializeEmu(char* homeFolder, void*, void*);
-	void __stdcall LoadROM(char* filename);
+	void __stdcall LoadROM(const char* filename);
 	void __stdcall Run();
 	void __stdcall Stop();
 }

TestHelper/TestHelper.cpp

@@ -32,6 +32,8 @@
 #include "../Utilities/Timer.h"
 #include "../Core/MessageManager.h"
 
+using namespace std;
+
 typedef void (__stdcall *NotificationListenerCallback)(ConsoleNotificationType);
 
 class InteropNotificationListener : public INotificationListener
@@ -50,7 +52,7 @@ public:
 };
 
 extern "C" {
-	void __stdcall InitializeEmu(char* homeFolder, void*, void*);
+	void __stdcall InitializeEmu(const char* homeFolder, void*, void*);
 	int __stdcall RomTestRun(char* filename);
 	void __stdcall LoadROM(char* filename);
 	void __stdcall Run();
@@ -65,7 +67,7 @@ vector<string> failedTests;
 SimpleLock lock;
 Timer timer;
 
-void OnNotificationReceived(ConsoleNotificationType type)
+void _stdcall OnNotificationReceived(ConsoleNotificationType type)
 {
 	if(type == ConsoleNotificationType::GameLoaded) {
 		runThread = new std::thread(Run);
@@ -81,7 +83,7 @@ void RunTest()
 {
 	while(true) {
 		lock.Acquire();
-		int index = testIndex++;
+		size_t index = testIndex++;
 		lock.Release();
 
 		if(index < testFilenames.size()) {
@@ -93,11 +95,12 @@ void RunTest()
 			std::cout << std::to_string(index) << ") " << filename << std::endl;
 			lock.Release();
 
-			if(std::system(command.c_str()) != 1) {
+			int failedFrames = std::system(command.c_str());
+			if(failedFrames != 0) {
 				//Test failed
 				lock.Acquire();
 				failedTests.push_back(filename);
-				std::cout << "  ****  " << std::to_string(index) << ") " << filename << " failed" << std::endl;
+				std::cout << "  ****  " << std::to_string(index) << ") " << filename << " failed (" << failedFrames << ")" << std::endl;
 				lock.Release();
 			}
 		} else {
@@ -108,13 +111,14 @@ void RunTest()
 
 int main(int argc, char* argv[])
 {
-	using namespace std;
+	wchar_t path[MAX_PATH];
+	SHGetFolderPath(NULL, CSIDL_MYDOCUMENTS, NULL, SHGFP_TYPE_CURRENT, path);
+	string mesenFolder = FolderUtilities::CombinePath(utf8::utf8::encode(path), "Mesen");
+
 	if(argc <= 2) {
 		string testFolder;
 		if(argc == 1) {
-			wchar_t path[MAX_PATH];
-			SHGetFolderPath(NULL, CSIDL_MYDOCUMENTS, NULL, SHGFP_TYPE_CURRENT, path);
-			testFolder = utf8::utf8::encode(path) + "\\Mesen\\Tests";
+			testFolder = FolderUtilities::CombinePath(mesenFolder, "Tests");
 		} else {
 			testFolder = argv[1];
 		}
@@ -135,12 +139,16 @@ int main(int argc, char* argv[])
 			delete testThreads[i];
 		}
 
-		std::cout << std::endl << std::endl;
-		std::cout << "------------" << std::endl;
-		std::cout << "Failed tests" << std::endl;
-		std::cout << "------------" << std::endl;
-		for(string failedTest : failedTests) {
-			std::cout << failedTest << std::endl;
+		if(!failedTests.empty()) {
+			std::cout << std::endl << std::endl;
+			std::cout << "------------" << std::endl;
+			std::cout << "Failed tests" << std::endl;
+			std::cout << "------------" << std::endl;
+			for(string failedTest : failedTests) {
+				std::cout << failedTest << std::endl;
+			}
+		} else {
+			std::cout << std::endl << std::endl << "All tests passed.";
 		}
 		std::cout << std::endl << std::endl << "Elapsed time: " << (timer.GetElapsedMS() / 1000) << " seconds";
 
@@ -148,7 +156,9 @@ int main(int argc, char* argv[])
 	} else if(argc == 3) {
 		char* testFilename = argv[2];
 		RegisterNotificationCallback((NotificationListenerCallback)OnNotificationReceived);
-		InitializeEmu("C:\\Windows\\Temp\\Mesen", nullptr, nullptr);
+
+		InitializeEmu(mesenFolder.c_str(), nullptr, nullptr);
+
 		int result = RomTestRun(testFilename);
 		if(runThread != nullptr) {
 			runThread->join();