Tests: Added command line utility to run all tests on multiple threads at once (No GUI) to speed up test runs

This commit is contained in:
Souryo 2015-12-29 16:27:37 -05:00
parent b6fc3471f8
commit 10fda05e32
4 changed files with 357 additions and 0 deletions

13
NES.sln
View File

@ -32,6 +32,11 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PGOHelper", "PGOHelper\PGOH
EndProject EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BlipBuffer", "BlipBuffer\BlipBuffer.vcxproj", "{CF35D78C-F710-41D2-968F-C46ACCFF6F07}" Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BlipBuffer", "BlipBuffer\BlipBuffer.vcxproj", "{CF35D78C-F710-41D2-968F-C46ACCFF6F07}"
EndProject EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "TestHelper", "TestHelper\TestHelper.vcxproj", "{2A607369-8B5D-494A-9E40-C5DC8D821AA3}"
ProjectSection(ProjectDependencies) = postProject
{37749BB2-FA78-4EC9-8990-5628FC0BBA19} = {37749BB2-FA78-4EC9-8990-5628FC0BBA19}
EndProjectSection
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64 Debug|x64 = Debug|x64
@ -95,6 +100,14 @@ Global
{CF35D78C-F710-41D2-968F-C46ACCFF6F07}.Release|x64.ActiveCfg = Release|x64 {CF35D78C-F710-41D2-968F-C46ACCFF6F07}.Release|x64.ActiveCfg = Release|x64
{CF35D78C-F710-41D2-968F-C46ACCFF6F07}.Release|x86.ActiveCfg = Release|Win32 {CF35D78C-F710-41D2-968F-C46ACCFF6F07}.Release|x86.ActiveCfg = Release|Win32
{CF35D78C-F710-41D2-968F-C46ACCFF6F07}.Release|x86.Build.0 = Release|Win32 {CF35D78C-F710-41D2-968F-C46ACCFF6F07}.Release|x86.Build.0 = Release|Win32
{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Debug|x64.ActiveCfg = Debug|x64
{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Debug|x64.Build.0 = Debug|x64
{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Debug|x86.ActiveCfg = Debug|Win32
{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Debug|x86.Build.0 = Debug|Win32
{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Release|x64.ActiveCfg = Release|x64
{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Release|x64.Build.0 = Release|x64
{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Release|x86.ActiveCfg = Release|Win32
{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Release|x86.Build.0 = Release|Win32
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE

161
TestHelper/TestHelper.cpp Normal file
View File

@ -0,0 +1,161 @@
#if _WIN64 || __x86_64__ || __ppc64__
#define ENVIRONMENT64
#else
#define ENVIRONMENT32
#endif
#ifdef ENVIRONMENT32
#ifdef _DEBUG
#define MESEN_LIBRARY_PATH "../bin/x86/Debug/"
#else
#define MESEN_LIBRARY_PATH "../bin/x86/Release/"
#endif
#else
#ifdef _DEBUG
#define MESEN_LIBRARY_PATH "../bin/x64/Debug/"
#else
#define MESEN_LIBRARY_PATH "../bin/x64/Release/"
#endif
#endif
#pragma comment(lib, MESEN_LIBRARY_PATH"Utilities.lib")
#include <Windows.h>
#include <Shlobj.h>
#include <iostream>
#include <thread>
#include <vector>
#include <string>
#include <thread>
#include "../Utilities/FolderUtilities.h"
#include "../Utilities/SimpleLock.h"
#include "../Utilities/Timer.h"
#include "../Core/MessageManager.h"
typedef void (__stdcall *NotificationListenerCallback)(ConsoleNotificationType);
class InteropNotificationListener : public INotificationListener
{
NotificationListenerCallback _callback;
public:
InteropNotificationListener(NotificationListenerCallback callback)
{
_callback = callback;
}
void ProcessNotification(ConsoleNotificationType type, void* parameter)
{
_callback((ConsoleNotificationType)type);
}
};
extern "C" {
void __stdcall InitializeEmu(char* homeFolder, void*, void*);
int __stdcall RomTestRun(char* filename);
void __stdcall LoadROM(char* filename);
void __stdcall Run();
void __stdcall Stop();
INotificationListener* __stdcall RegisterNotificationCallback(NotificationListenerCallback callback);
}
std::thread *runThread = nullptr;
std::atomic<int> testIndex;
vector<string> testFilenames;
vector<string> failedTests;
SimpleLock lock;
Timer timer;
void OnNotificationReceived(ConsoleNotificationType type)
{
if(type == ConsoleNotificationType::GameLoaded) {
runThread = new std::thread(Run);
}
}
void RunEmu()
{
Run();
}
void RunTest()
{
while(true) {
lock.Acquire();
int index = testIndex++;
lock.Release();
if(index < testFilenames.size()) {
string filepath = testFilenames[index];
string filename = FolderUtilities::GetFilename(filepath, false);
string command = "TestHelper.exe /testrom \"" + filepath + "\"";
lock.Acquire();
std::cout << std::to_string(index) << ") " << filename << std::endl;
lock.Release();
if(std::system(command.c_str()) != 1) {
//Test failed
lock.Acquire();
failedTests.push_back(filename);
std::cout << " **** " << std::to_string(index) << ") " << filename << " failed" << std::endl;
lock.Release();
}
} else {
break;
}
}
}
int main(int argc, char* argv[])
{
using namespace std;
if(argc <= 2) {
string testFolder;
if(argc == 1) {
wchar_t path[MAX_PATH];
SHGetFolderPath(NULL, CSIDL_MYDOCUMENTS, NULL, SHGFP_TYPE_CURRENT, path);
testFolder = utf8::utf8::encode(path) + "\\Mesen\\Tests";
} else {
testFolder = argv[1];
}
vector<std::thread*> testThreads;
testFilenames = FolderUtilities::GetFilesInFolder(testFolder, "*.mtp", true);
testIndex = 0;
timer.Reset();
for(int i = 0; i < 4; i++) {
std::thread *testThread = new std::thread(RunTest);
testThreads.push_back(testThread);
}
for(int i = 0; i < 4; i++) {
testThreads[i]->join();
delete testThreads[i];
}
std::cout << std::endl << std::endl;
std::cout << "------------" << std::endl;
std::cout << "Failed tests" << std::endl;
std::cout << "------------" << std::endl;
for(string failedTest : failedTests) {
std::cout << failedTest << std::endl;
}
std::cout << std::endl << std::endl << "Elapsed time: " << (timer.GetElapsedMS() / 1000) << " seconds";
std::getchar();
} else if(argc == 3) {
char* testFilename = argv[2];
RegisterNotificationCallback((NotificationListenerCallback)OnNotificationReceived);
InitializeEmu("C:\\Windows\\Temp\\Mesen", nullptr, nullptr);
int result = RomTestRun(testFilename);
if(runThread != nullptr) {
runThread->join();
delete runThread;
}
return result;
}
return 0;
}

View File

@ -0,0 +1,169 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{2A607369-8B5D-494A-9E40-C5DC8D821AA3}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>TestHelper</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)\bin\$(PlatformTarget)\$(Configuration)\</OutDir>
<IntDir>obj\$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<IntDir>obj\$(Platform)\$(Configuration)\</IntDir>
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)\bin\$(PlatformTarget)\$(Configuration)\</OutDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)\bin\$(PlatformTarget)\$(Configuration)\</OutDir>
<IntDir>obj\$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<IntDir>obj\$(Platform)\$(Configuration)\</IntDir>
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)\bin\$(PlatformTarget)\$(Configuration)\</OutDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="TestHelper.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\InteropDLL\InteropDLL.vcxproj">
<Project>{37749bb2-fa78-4ec9-8990-5628fc0bba19}</Project>
<Private>false</Private>
<ReferenceOutputAssembly>true</ReferenceOutputAssembly>
<CopyLocalSatelliteAssemblies>false</CopyLocalSatelliteAssemblies>
<LinkLibraryDependencies>true</LinkLibraryDependencies>
<UseLibraryDependencyInputs>true</UseLibraryDependencyInputs>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Source Files">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="TestHelper.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
</Project>