From 8c9627075281145c02889510c9193d2f18514fb0 Mon Sep 17 00:00:00 2001 From: Erik Abair Date: Tue, 15 Mar 2022 10:37:04 -0700 Subject: [PATCH] Change Test to TestBase and support multiple test classes --- Dockerfile | 4 ++- README.md | 3 ++ xemutest/__init__.py | 1 - xemutest/__main__.py | 48 +++++++++++++++++++++++++----- xemutest/{test.py => test_base.py} | 29 ++++++++++-------- xemutest/test_xbe.py | 21 +++++++++++++ 6 files changed, 83 insertions(+), 23 deletions(-) rename xemutest/{test.py => test_base.py} (89%) create mode 100644 xemutest/test_xbe.py diff --git a/Dockerfile b/Dockerfile index f714669..2328e5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,11 @@ # FROM ghcr.io/xboxdev/nxdk AS data RUN mkdir /data + COPY test-xbe /test-xbe +RUN mkdir /data/TestXBE RUN /usr/src/nxdk/docker_entry.sh make -C /test-xbe -RUN cp /test-xbe/tester.iso /data +RUN cp /test-xbe/tester.iso /data/TestXBE/ # # Build base test container image diff --git a/README.md b/README.md index db8b8cd..79b842f 100644 --- a/README.md +++ b/README.md @@ -56,3 +56,6 @@ Containers are a great solution generally, but they aren't available on every platform that xemu runs on. The tester can be run outside of a container, provided you have set up the environment correctly. It is a goal for this project to support running the tests natively on all platforms. + +See `xemutest/__main__.py` for a description of arguments that may be used to +customize native behavior. diff --git a/xemutest/__init__.py b/xemutest/__init__.py index 6d5150c..e69de29 100644 --- a/xemutest/__init__.py +++ b/xemutest/__init__.py @@ -1 +0,0 @@ -from .test import Test diff --git a/xemutest/__main__.py b/xemutest/__main__.py index d88c781..1ff13f7 100644 --- a/xemutest/__main__.py +++ b/xemutest/__main__.py @@ -1,7 +1,8 @@ -import logging import argparse - -from xemutest import Test +import importlib +import inspect +import logging +import os logging.basicConfig(level=logging.INFO) @@ -12,18 +13,49 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument('private', help='Path to private data files') ap.add_argument('results', help='Path to directory where results should go') + ap.add_argument('--data', help='Path to test data (e.g., disc images)') + ap.add_argument('--xemu', help='Path to the xemu binary') args = ap.parse_args() + tests = [] result = True - tests = [Test] - for i, test_cls in enumerate(tests): + for path in os.listdir(os.path.dirname(__file__)): + if not path.startswith("test_") or path == "test_base.py": + continue + if not path.endswith(".py"): + continue + + module = importlib.import_module(path[:-3]) + for test_name, test_class in inspect.getmembers(module, inspect.isclass): + if test_name.startswith("Test"): + tests.append((test_name, test_class)) + + private_path = os.path.abspath(os.path.expanduser(args.private)) + results_base = os.path.abspath(os.path.expanduser(args.results)) + if args.data: + test_data_root = os.path.expanduser(args.data) + else: + test_data_root = os.path.join(os.path.dirname(__file__), 'data') + xemu_path = args.xemu + if xemu_path: + xemu_path = os.path.abspath(os.path.expanduser(xemu_path)) + for i, (test_name, test_cls) in enumerate(tests): log.info('Test %d', i) log.info('-'*40) + + test_results = os.path.join(results_base, test_name) + test_data = os.path.join(test_data_root, test_name) try: - test_cls(args.private, args.results).run() - log.info('Test %d passed!', i) + test = test_cls(private_path, test_results, test_data, xemu_path) except: - log.exception('Test %d failed!', i) + log.exception('Test %d - %s setup failed!', i, test_name) + result = False + continue + try: + test.run() + log.info('Test %d - %s passed!', i, test_name) + except: + log.exception('Test %d - %s failed!', i, test_name) result = False exit(0 if result else 1) diff --git a/xemutest/test.py b/xemutest/test_base.py similarity index 89% rename from xemutest/test.py rename to xemutest/test_base.py index f5f4c23..c109b22 100755 --- a/xemutest/test.py +++ b/xemutest/test_base.py @@ -20,9 +20,9 @@ if platform.system() == 'Windows': log = logging.getLogger(__file__) -class Test: +class TestBase: """ - Test provides a basic framework that: + Provides a basic framework that: - Starts FFMPEG to record footage of xemu while it runs - Launches xemu with an test XBE loaded from a disc image - Waits for xemu to shutdown or timeout @@ -31,22 +31,22 @@ class Test: Tester runs in current working directory and will generate some working files. """ - def __init__(self, private_path: str, results_path: str): + def __init__(self, private_path: str, results_path: str, iso_path: str, xemu_path: Optional[str]): cur_dir = os.getcwd() - if platform.system() == 'Windows': - self.xemu_path = os.path.join(cur_dir, 'xemu.exe') - else: - self.xemu_path = 'xemu' - test_data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data')) - if not os.path.isdir(test_data_path): - raise FileNotFoundError('Test data was not installed with the package. You need to build it.') + if not xemu_path: + if platform.system() == 'Windows': + self.xemu_path = os.path.join(cur_dir, 'xemu.exe') + else: + self.xemu_path = 'xemu' + else: + self.xemu_path = xemu_path self.flash_path = os.path.join(private_path, 'bios.bin') self.mcpx_path = os.path.join(private_path, 'mcpx.bin') self.hdd_path = os.path.join(cur_dir, 'test.img') self.mount_path = os.path.join(cur_dir, 'xemu-hdd-mount') - self.iso_path = os.path.join(test_data_path, 'tester.iso') + self.iso_path = iso_path self.results_in_path = os.path.join(self.mount_path, 'results') self.results_out_path = results_path self.video_capture_path = os.path.join(self.results_out_path, 'capture.mp4') @@ -175,8 +175,11 @@ class Test: # Nothing to do def analyze_results(self): - with open(os.path.join(self.results_out_path, 'results.txt')) as f: - assert(f.read().strip() == 'Success') + """Validate any files retrieved from the HDD. + + This method should be implemented by the subclass to confirm that the output of the test matches expectations. + """ + pass def run(self): os.makedirs(self.results_out_path, exist_ok=True) diff --git a/xemutest/test_xbe.py b/xemutest/test_xbe.py new file mode 100644 index 0000000..41d26a0 --- /dev/null +++ b/xemutest/test_xbe.py @@ -0,0 +1,21 @@ +"""Test harness for test-xbe.""" + +import os +from typing import Optional + +import test_base + +class TestXBE(test_base.TestBase): + """Runs test-xbe and validates output.""" + + def __init__(self, private_path: str, results_path: str, test_data_path: str, xemu_path: Optional[str]): + iso_path = os.path.join(test_data_path, 'tester.iso') + if not os.path.isfile(iso_path): + raise FileNotFoundError('Test data was not installed with the package. You need to build it and copy ' + f'to {test_data_path}.') + + super().__init__(private_path, results_path, iso_path, xemu_path) + + def analyze_results(self): + with open(os.path.join(self.results_out_path, 'results.txt')) as f: + assert(f.read().strip() == 'Success')