Change Test to TestBase and support multiple test classes

Dockerfile

@@ -3,9 +3,11 @@
 #
 FROM ghcr.io/xboxdev/nxdk AS data
 RUN mkdir /data
+
 COPY test-xbe /test-xbe
+RUN mkdir /data/TestXBE
 RUN /usr/src/nxdk/docker_entry.sh make -C /test-xbe
-RUN cp /test-xbe/tester.iso /data
+RUN cp /test-xbe/tester.iso /data/TestXBE/
 
 #
 # Build base test container image

README.md

@@ -56,3 +56,6 @@ Containers are a great solution generally, but they aren't available on every
 platform that xemu runs on. The tester can be run outside of a container,
 provided you have set up the environment correctly. It is a goal for this project
 to support running the tests natively on all platforms.
+
+See `xemutest/__main__.py` for a description of arguments that may be used to
+customize native behavior.

xemutest/__init__.py

@@ -1 +0,0 @@
-from .test import Test

xemutest/__main__.py

@@ -1,7 +1,8 @@
-import logging
 import argparse
-
-from xemutest import Test
+import importlib
+import inspect
+import logging
+import os
 
 
 logging.basicConfig(level=logging.INFO)
@@ -12,18 +13,49 @@ def main():
 	ap = argparse.ArgumentParser()
 	ap.add_argument('private', help='Path to private data files')
 	ap.add_argument('results', help='Path to directory where results should go')
+	ap.add_argument('--data', help='Path to test data (e.g., disc images)')
+	ap.add_argument('--xemu', help='Path to the xemu binary')
 	args = ap.parse_args()
 
+	tests = []
 	result = True
-	tests = [Test]
-	for i, test_cls in enumerate(tests):
+	for path in os.listdir(os.path.dirname(__file__)):
+		if not path.startswith("test_") or path == "test_base.py":
+			continue
+		if not path.endswith(".py"):
+			continue
+
+		module = importlib.import_module(path[:-3])
+		for test_name, test_class in inspect.getmembers(module, inspect.isclass):
+			if test_name.startswith("Test"):
+				tests.append((test_name, test_class))
+
+	private_path = os.path.abspath(os.path.expanduser(args.private))
+	results_base = os.path.abspath(os.path.expanduser(args.results))
+	if args.data:
+		test_data_root = os.path.expanduser(args.data)
+	else:
+		test_data_root = os.path.join(os.path.dirname(__file__), 'data')
+	xemu_path = args.xemu
+	if xemu_path:
+		xemu_path = os.path.abspath(os.path.expanduser(xemu_path))
+	for i, (test_name, test_cls) in enumerate(tests):
 		log.info('Test %d', i)
 		log.info('-'*40)
+		
+		test_results = os.path.join(results_base, test_name)
+		test_data = os.path.join(test_data_root, test_name)
 		try:
-			test_cls(args.private, args.results).run()
-			log.info('Test %d passed!', i)
+			test = test_cls(private_path, test_results, test_data, xemu_path)
 		except:
-			log.exception('Test %d failed!', i)
+			log.exception('Test %d - %s setup failed!', i, test_name)
+			result = False
+			continue
+		try:
+			test.run()
+			log.info('Test %d - %s passed!', i, test_name)
+		except:
+			log.exception('Test %d - %s failed!', i, test_name)
 			result = False
 
 	exit(0 if result else 1)

xemutest/test_base.py

@@ -20,9 +20,9 @@ if platform.system() == 'Windows':
 log = logging.getLogger(__file__)
 
 
-class Test:
+class TestBase:
 	"""
-	Test provides a basic framework that:
+	Provides a basic framework that:
 	- Starts FFMPEG to record footage of xemu while it runs
 	- Launches xemu with an test XBE loaded from a disc image
 	- Waits for xemu to shutdown or timeout
@@ -31,22 +31,22 @@ class Test:
 	Tester runs in current working directory and will generate some working files.
 	"""
 
-	def __init__(self, private_path: str, results_path: str):
+	def __init__(self, private_path: str, results_path: str, iso_path: str, xemu_path: Optional[str]):
 		cur_dir = os.getcwd()
+
+		if not xemu_path:
 			if platform.system() == 'Windows':
 				self.xemu_path = os.path.join(cur_dir, 'xemu.exe')
 			else:
 				self.xemu_path = 'xemu'
-
-		test_data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data'))
-		if not os.path.isdir(test_data_path):
-			raise FileNotFoundError('Test data was not installed with the package. You need to build it.')
+		else:
+			self.xemu_path = xemu_path
 
 		self.flash_path         = os.path.join(private_path, 'bios.bin')
 		self.mcpx_path          = os.path.join(private_path, 'mcpx.bin')
 		self.hdd_path           = os.path.join(cur_dir, 'test.img')
 		self.mount_path         = os.path.join(cur_dir, 'xemu-hdd-mount')
-		self.iso_path           = os.path.join(test_data_path, 'tester.iso')
+		self.iso_path           = iso_path
 		self.results_in_path    = os.path.join(self.mount_path, 'results')
 		self.results_out_path   = results_path
 		self.video_capture_path = os.path.join(self.results_out_path, 'capture.mp4')
@@ -175,8 +175,11 @@ class Test:
 		# Nothing to do
 
 	def analyze_results(self):
-		with open(os.path.join(self.results_out_path, 'results.txt')) as f:
-			assert(f.read().strip() == 'Success')
+		"""Validate any files retrieved from the HDD.
+
+		This method should be implemented by the subclass to confirm that the output of the test matches expectations.
+		"""
+		pass
 
 	def run(self):
 		os.makedirs(self.results_out_path, exist_ok=True)

xemutest/test_xbe.py

@@ -0,0 +1,21 @@
+"""Test harness for test-xbe."""
+
+import os
+from typing import Optional
+
+import test_base
+
+class TestXBE(test_base.TestBase):
+    """Runs test-xbe and validates output."""
+
+    def __init__(self, private_path: str, results_path: str, test_data_path: str, xemu_path: Optional[str]):
+        iso_path = os.path.join(test_data_path, 'tester.iso')
+        if not os.path.isfile(iso_path):
+            raise FileNotFoundError('Test data was not installed with the package. You need to build it and copy '
+                                    f'to {test_data_path}.')
+
+        super().__init__(private_path, results_path, iso_path, xemu_path)
+
+    def analyze_results(self):
+        with open(os.path.join(self.results_out_path, 'results.txt')) as f:
+            assert(f.read().strip() == 'Success')