xemu-project/nxdk_pgraph_tests
1
0
Fork 0
mirror of https://github.com/xemu-project/nxdk_pgraph_tests.git synced 2024-11-23 10:09:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Adds automatic crash detection and interactive avoidance.

Browse Source
This commit is contained in:
Erik Abair 2022-07-13 13:39:14 -07:00
parent 00a4ebe87f
commit 337757a9fa
9 changed files with 334 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/build.yml vendored
View File

@ -36,7 +36,11 @@ jobs:
- name: Compile
run: |
cd nxdk_pgraph_tests
make -j $(grep -c processor /proc/cpuinfo) ENABLE_PROGRESS_LOG=y RUNTIME_CONFIG_PATH="e:/nxdk_pgraph_tests/pgraph_tests.cnf" DUMP_CONFIG_FILE=y
make -j $(grep -c processor /proc/cpuinfo) \
ENABLE_PROGRESS_LOG=y \
ENABLE_INTERACTIVE_CRASH_AVOIDANCE=y \
RUNTIME_CONFIG_PATH="e:/nxdk_pgraph_tests/pgraph_tests.cnf" \
DUMP_CONFIG_FILE=y
- name: Create release
if: github.ref == 'refs/heads/main'
uses: "marvinpinto/action-automatic-releases@latest"

10
Makefile
View File

@ -178,6 +178,16 @@ ifeq ($(ENABLE_PROGRESS_LOG),y)
CXXFLAGS += -DENABLE_PROGRESS_LOG
endif
# Uses the result of the last progress log to ask the user whether tests that appeared to crash historically should be
# skipped.
ENABLE_INTERACTIVE_CRASH_AVOIDANCE ?= n
ifeq ($(ENABLE_INTERACTIVE_CRASH_AVOIDANCE),y)
ifneq ($(ENABLE_PROGRESS_LOG),y)
$(error ENABLE_INTERACTIVE_CRASH_AVOIDANCE may not be enabled without ENABLE_PROGRESS_LOG)
endif
CXXFLAGS += -DENABLE_INTERACTIVE_CRASH_AVOIDANCE
endif
# Causes a diff of the nv2a PGRAPH registers to be done between the start and end of each test in order to detect state
# leakage. Output is logged to XBDM and will be written into the progress log if it is enabled.
ENABLE_PGRAPH_REGION_DIFF ?= n

105
src/main.cpp
View File

@ -89,6 +89,8 @@ static bool get_writable_output_directory(std::string& xbe_root_directory);
static bool get_test_output_path(std::string& test_output_directory);
static void dump_config_file(const std::string& config_file_path,
const std::vector<std::shared_ptr<TestSuite>>& test_suites);
static bool discover_historical_crashes(const std::string& log_file_path,
std::map<std::string, std::set<std::string>>& crashes);
static bool process_config(const char* config_file_path, std::vector<std::shared_ptr<TestSuite>>& test_suites);
/* Main program function */
@ -135,12 +137,26 @@ int main() {
TestHost::EnsureFolderExists(test_output_directory);
std::map<std::string, std::set<std::string>> historical_crashes;
#ifdef ENABLE_PROGRESS_LOG
{
std::string log_file = test_output_directory + "\\" + kLogFileName;
#ifdef ENABLE_INTERACTIVE_CRASH_AVOIDANCE
discover_historical_crashes(log_file, historical_crashes);
#endif
DeleteFile(log_file.c_str());
Logger::Initialize(log_file, true);
if (!historical_crashes.empty()) {
Logger::Log() << "<HistoricalCrashes>" << std::endl;
for (auto& suite_tests : historical_crashes) {
for (auto& test : suite_tests.second) {
Logger::Log() << "Crash? " << suite_tests.first << "::" << test << std::endl;
}
}
Logger::Log() << "</HistoricalCrashes>" << std::endl;
}
}
#endif
@ -157,7 +173,16 @@ int main() {
process_config("d:\\pgraph_tests.cnf", test_suites);
}
TestDriver driver(host, test_suites, kFramebufferWidth, kFramebufferHeight);
if (!historical_crashes.empty()) {
for (auto& suite : test_suites) {
auto crash_info = historical_crashes.find(suite->Name());
if (crash_info != historical_crashes.end()) {
suite->SetSuspectedCrashes(crash_info->second);
}
}
}
TestDriver driver(host, test_suites, kFramebufferWidth, kFramebufferHeight, !historical_crashes.empty());
driver.Run();
#ifdef ENABLE_SHUTDOWN
@ -249,6 +274,84 @@ static void dump_config_file(const std::string& config_file_path,
}
}
static bool discover_historical_crashes(const std::string& log_file_path,
std::map<std::string, std::set<std::string>>& crashes) {
crashes.clear();
if (!ensure_drive_mounted(log_file_path[0])) {
return false;
}
std::string dos_style_path = log_file_path;
std::replace(dos_style_path.begin(), dos_style_path.end(), '/', '\\');
std::ifstream log_file(dos_style_path.c_str());
if (!log_file) {
return false;
}
std::string last_test_suite;
std::string last_test_name;
std::string line;
auto add_crash = [&crashes](const std::string& suite, const std::string& test) {
auto toskip = crashes.find(suite);
if (toskip == crashes.end()) {
crashes[suite] = {test};
} else {
toskip->second.emplace(test);
}
};
while (std::getline(log_file, line)) {
PrintMsg("'%s'\n", line.c_str());
if (!line.compare(0, 7, "Crash? ")) {
line = line.substr(7);
auto delimiter = line.find("::");
add_crash(line.substr(0, delimiter), line.substr(delimiter + 2));
continue;
}
if (!line.compare(0, 9, "Starting ")) {
if (!last_test_suite.empty()) {
PrintMsg("Potential crash: '%s' '%s'\n", last_test_suite.c_str(), last_test_name.c_str());
add_crash(last_test_suite, last_test_name);
}
line = line.substr(9);
auto delimiter = line.find("::");
last_test_suite = line.substr(0, delimiter);
last_test_name = line.substr(delimiter + 2);
continue;
}
if (!line.compare(0, 13, " Completed '")) {
std::string test_name = line.substr(13);
auto delimiter = test_name.rfind("' in ");
test_name = test_name.substr(0, delimiter);
if (test_name == last_test_name) {
auto crash_suite = crashes.find(last_test_suite);
if (crash_suite != crashes.end()) {
crash_suite->second.erase(last_test_name);
}
last_test_suite.clear();
last_test_name.clear();
continue;
}
PrintMsg("Completed line mismatch: '%s' '%s' but completed: '%s'\n", last_test_suite.c_str(),
last_test_name.c_str(), test_name.c_str());
}
PrintMsg("Unprocessed log line '%s'\n", line.c_str());
}
if (!last_test_suite.empty()) {
PrintMsg("Potential crash: '%s' '%s'\n", last_test_suite.c_str(), last_test_name.c_str());
add_crash(last_test_suite, last_test_name);
}
return true;
}
static bool process_config(const char* config_file_path, std::vector<std::shared_ptr<TestSuite>>& test_suites) {
if (!ensure_drive_mounted(config_file_path[0])) {
PrintMsg("Ignoring missing config at %s\n", config_file_path);

136
src/menu_item.cpp
View File

@ -3,8 +3,10 @@
#include <pbkit/pbkit.h>
#include <chrono>
#include <memory>
#include <utility>
#include "debug_output.h"
#include "tests/test_suite.h"
#ifdef AUTORUN_IMMEDIATELY
@ -321,3 +323,137 @@ void MenuItemRoot::CursorRight() {
timer_cancelled = true;
MenuItem::CursorRight();
}
struct MenuItemOption : public MenuItem {
MenuItemOption(const std::string &name, std::function<void(const MenuItemOption &)> on_apply);
MenuItemOption(const std::string &label, const std::vector<std::string> &values,
std::function<void(const MenuItemOption &)> on_apply);
inline void UpdateName() { name = label + ": " + values[current_option]; }
void Activate() override;
void CursorLeft() override;
void CursorRight() override;
std::string label;
std::vector<std::string> values;
uint32_t current_option = 0;
std::function<void(const MenuItemOption &)> on_apply;
};
MenuItemOption::MenuItemOption(const std::string &name, std::function<void(const MenuItemOption &)> on_apply)
: MenuItem(name, 0, 0), on_apply(on_apply) {}
MenuItemOption::MenuItemOption(const std::string &label, const std::vector<std::string> &values,
std::function<void(const MenuItemOption &)> on_apply)
: MenuItem("", 0, 0), label(label), values(values), on_apply(on_apply) {
ASSERT(!values.empty())
UpdateName();
}
void MenuItemOption::Activate() {
current_option = (current_option + 1) % values.size();
UpdateName();
}
void MenuItemOption::CursorLeft() {
current_option = (current_option - 1) % values.size();
UpdateName();
}
void MenuItemOption::CursorRight() { Activate(); }
MenuItemOptions::MenuItemOptions(const std::vector<std::shared_ptr<TestSuite>> &suites, std::function<void()> on_exit,
uint32_t width, uint32_t height)
: MenuItem("<<options>>", width, height), on_exit(std::move(on_exit)) {
submenu.push_back(
std::make_shared<MenuItemOption>("Accept", [this](const MenuItemOption &_ignored) { this->on_exit(); }));
{
constexpr uint32_t OPT_SKIP = 0;
constexpr uint32_t OPT_RUN = 1;
std::vector<std::string> values = {"Skip", "Run"};
auto on_apply = [suites](const MenuItemOption &opt) {
if (opt.current_option == OPT_SKIP) {
for (auto &suite : suites) {
suite->SetSuspectedCrashHandlingMode(TestSuite::SuspectedCrashHandling::SKIP_ALL);
}
} else if (opt.current_option == OPT_RUN) {
for (auto &suite : suites) {
suite->SetSuspectedCrashHandlingMode(TestSuite::SuspectedCrashHandling::RUN_ALL);
}
}
};
submenu.push_back(std::make_shared<MenuItemOption>("Previous crashes", values, on_apply));
}
start_time = std::chrono::high_resolution_clock::now();
}
void MenuItemOptions::Draw() {
if (!timer_cancelled) {
auto now = std::chrono::high_resolution_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - start_time).count();
if (elapsed > kAutoTestAllTimeoutMilliseconds) {
on_exit();
return;
}
char run_all[128] = {0};
snprintf(run_all, 127, "Accept (automatic in %d ms)", kAutoTestAllTimeoutMilliseconds - elapsed);
submenu[0]->name = run_all;
} else {
submenu[0]->name = "Accept";
}
MenuItem::Draw();
}
void MenuItemOptions::Activate() {
timer_cancelled = true;
if (cursor_position == 0) {
for (auto i = 1; i < submenu.size(); ++i) {
const auto &item = *(reinterpret_cast<MenuItemOption *>(submenu[i].get()));
item.on_apply(item);
}
on_exit();
return;
}
MenuItem::Activate();
}
void MenuItemOptions::ActivateCurrentSuite() {
timer_cancelled = true;
MenuItem::ActivateCurrentSuite();
}
bool MenuItemOptions::Deactivate() {
timer_cancelled = true;
on_exit();
return false;
}
void MenuItemOptions::CursorUp() {
timer_cancelled = true;
MenuItem::CursorUp();
}
void MenuItemOptions::CursorDown() {
timer_cancelled = true;
MenuItem::CursorDown();
}
void MenuItemOptions::CursorLeft() {
timer_cancelled = true;
if (cursor_position > 0) {
submenu[cursor_position]->CursorLeft();
}
}
void MenuItemOptions::CursorRight() {
timer_cancelled = true;
if (cursor_position > 0) {
submenu[cursor_position]->CursorRight();
}
}

18
src/menu_item.h
View File

@ -115,4 +115,22 @@ struct MenuItemRoot : public MenuItem {
bool timer_cancelled{false};
};
struct MenuItemOptions : public MenuItem {
MenuItemOptions(const std::vector<std::shared_ptr<TestSuite>>& suites, std::function<void()> on_exit, uint32_t width,
uint32_t height);
void Draw() override;
void Activate() override;
void ActivateCurrentSuite() override;
bool Deactivate() override;
void CursorUp() override;
void CursorDown() override;
void CursorLeft() override;
void CursorRight() override;
std::function<void()> on_exit;
std::chrono::steady_clock::time_point start_time;
bool timer_cancelled{false};
};
#endif // NXDK_PGRAPH_TESTS_MENU_ITEM_H

37
src/test_driver.cpp
View File

@ -7,14 +7,23 @@
#include "menu_item.h"
TestDriver::TestDriver(TestHost &host, const std::vector<std::shared_ptr<TestSuite>> &test_suites,
uint32_t framebuffer_width, uint32_t framebuffer_height)
uint32_t framebuffer_width, uint32_t framebuffer_height, bool show_options_menu)
: test_host_(host),
test_suites_(test_suites),
framebuffer_width_(framebuffer_width),
framebuffer_height_(framebuffer_height) {
auto on_run_all = [this]() { RunAllTestsNonInteractive(); };
auto on_exit = [this]() { running_ = false; };
menu_ = std::make_shared<MenuItemRoot>(test_suites, on_run_all, on_exit, framebuffer_width, framebuffer_height);
root_menu_ = std::make_shared<MenuItemRoot>(test_suites, on_run_all, on_exit, framebuffer_width, framebuffer_height);
if (show_options_menu) {
auto on_options_exit = [this]() { active_menu_ = root_menu_; };
options_menu_ =
std::make_shared<MenuItemOptions>(test_suites, on_options_exit, framebuffer_width, framebuffer_height);
active_menu_ = options_menu_;
} else {
active_menu_ = root_menu_;
}
}
TestDriver::~TestDriver() {
@ -50,12 +59,12 @@ void TestDriver::Run() {
}
if (!test_host_.GetSaveResults()) {
menu_->SetBackgroundColor(0xFF3E1E1E);
active_menu_->SetBackgroundColor(0xFF3E1E1E);
} else {
menu_->SetBackgroundColor(0xFF1E1E1E);
active_menu_->SetBackgroundColor(0xFF1E1E1E);
}
menu_->Draw();
active_menu_->Draw();
Sleep(10);
}
@ -169,17 +178,17 @@ void TestDriver::ShowDebugMessageAndExit() {
running_ = false;
}
void TestDriver::OnBack() { menu_->Deactivate(); }
void TestDriver::OnBack() { active_menu_->Deactivate(); }
void TestDriver::OnStart() { menu_->Activate(); }
void TestDriver::OnStart() { active_menu_->Activate(); }
void TestDriver::OnBlack() { running_ = false; }
void TestDriver::OnA() { menu_->Activate(); }
void TestDriver::OnA() { active_menu_->Activate(); }
void TestDriver::OnB() { menu_->Deactivate(); }
void TestDriver::OnB() { active_menu_->Deactivate(); }
void TestDriver::OnX() { menu_->ActivateCurrentSuite(); }
void TestDriver::OnX() { active_menu_->ActivateCurrentSuite(); }
void TestDriver::OnY() {
bool save_results = !test_host_.GetSaveResults();
@ -187,10 +196,10 @@ void TestDriver::OnY() {
MenuItemTest::SetOneShotMode(save_results);
}
void TestDriver::OnUp() { menu_->CursorUp(); }
void TestDriver::OnUp() { active_menu_->CursorUp(); }
void TestDriver::OnDown() { menu_->CursorDown(); }
void TestDriver::OnDown() { active_menu_->CursorDown(); }
void TestDriver::OnLeft() { menu_->CursorLeft(); }
void TestDriver::OnLeft() { active_menu_->CursorLeft(); }
void TestDriver::OnRight() { menu_->CursorRight(); }
void TestDriver::OnRight() { active_menu_->CursorRight(); }

9
src/test_driver.h
View File

@ -5,7 +5,10 @@
#include <cstdint>
#include <list>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <vector>
#include "test_host.h"
@ -18,7 +21,7 @@ struct MenuItem;
class TestDriver {
public:
TestDriver(TestHost &host, const std::vector<std::shared_ptr<TestSuite>> &test_suites, uint32_t framebuffer_width,
uint32_t framebuffer_height);
uint32_t framebuffer_height, bool show_options_menu = false);
~TestDriver();
void Run();
@ -55,7 +58,9 @@ class TestDriver {
uint32_t framebuffer_height_;
TestHost &test_host_;
std::shared_ptr<MenuItem> menu_;
std::shared_ptr<MenuItem> active_menu_;
std::shared_ptr<MenuItem> root_menu_;
std::shared_ptr<MenuItem> options_menu_;
};
#endif // NXDK_PGRAPH_TESTS_TEST_DRIVER_H

19
src/tests/test_suite.cpp
View File

@ -48,6 +48,21 @@ void TestSuite::Run(const std::string& test_name) {
void TestSuite::RunAll() {
auto names = TestNames();
for (const auto& test_name : names) {
if (suspected_crashes_.find(test_name) != suspected_crashes_.end()) {
switch (suspected_crash_handling_mode_) {
case SuspectedCrashHandling::RUN_ALL:
break;
case SuspectedCrashHandling::SKIP_ALL:
continue;
case SuspectedCrashHandling::ASK:
// TODO: IMPLEMENT ASK MODE FOR CRASH HANDLING
ASSERT(!"TODO: IMPLEMENT ME");
break;
}
}
Run(test_name);
}
}
@ -252,11 +267,11 @@ void TestSuite::LogTestEnd(const std::string& test_name, std::chrono::steady_clo
auto now = std::chrono::high_resolution_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - start_time).count();
PrintMsg(" Completed %s %lums\n", test_name.c_str(), elapsed);
PrintMsg(" Completed '%s' in %lums\n", test_name.c_str(), elapsed);
#ifdef ENABLE_PROGRESS_LOG
if (allow_saving_) {
Logger::Log() << " Completed " << test_name << " " << elapsed << "ms" << std::endl;
Logger::Log() << " Completed '" << test_name << "' in " << elapsed << "ms" << std::endl;
}
#endif
}

14
src/tests/test_suite.h
View File

@ -4,6 +4,7 @@
#include <chrono>
#include <fstream>
#include <map>
#include <set>
#include <string>
#include <vector>
@ -12,6 +13,13 @@
class TestHost;
class TestSuite {
public:
enum class SuspectedCrashHandling {
SKIP_ALL,
RUN_ALL,
ASK,
};
public:
TestSuite(TestHost &host, std::string output_dir, std::string suite_name);
@ -21,6 +29,7 @@ class TestSuite {
virtual void Deinitialize();
void DisableTests(const std::vector<std::string> &tests_to_skip);
void SetSuspectedCrashes(const std::set<std::string> &test_names) { suspected_crashes_ = test_names; }
std::vector<std::string> TestNames() const;
void Run(const std::string &test_name);
@ -28,6 +37,7 @@ class TestSuite {
void RunAll();
void SetSavingAllowed(bool enable = true) { allow_saving_ = enable; }
void SetSuspectedCrashHandlingMode(SuspectedCrashHandling mode) { suspected_crash_handling_mode_ = mode; }
protected:
void SetDefaultTextureFormat() const;
@ -46,6 +56,10 @@ class TestSuite {
// Map of `test_name` to `void test()`
std::map<std::string, std::function<void()>> tests_{};
// Tests that have crashed historically.
std::set<std::string> suspected_crashes_{};
SuspectedCrashHandling suspected_crash_handling_mode_{SuspectedCrashHandling::RUN_ALL};
PGRAPHDiffToken pgraph_diff_;
};