Reorg, Generate HDD image, Windows compatibility

.github/workflows/build_docker_image.yml

@@ -8,7 +8,7 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  main:
+  build-container:
     name: Build and Publish Image
     if: github.repository_owner == 'mborgerson'
     runs-on: ubuntu-latest
@@ -28,7 +28,7 @@ jobs:
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v1
-        if: github.event_name != 'pull_request'
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -38,6 +38,51 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+  build-wheel:
+    runs-on: ubuntu-latest
+    needs: [build-container]
+    steps:
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+
+      - name: Clone Tree
+        uses: actions/checkout@v2
+
+      - name: Build Test Data
+        run: bash ./scripts/build_test_data.sh
+
+      - name: Build wheel
+        run: |
+          pip install wheel
+          pip install .
+          mkdir /tmp/wheels
+          pip wheel -w /tmp/wheels .
+          mv /tmp/wheels/xemutest-*.whl .
+
+      - name: Get package info
+        run: |
+          echo "TAG_NAME=wheel-$(date -u +'%Y%m%d%H%M')" >> $GITHUB_ENV
+          echo "WHEEL_FILENAME=$(ls *.whl)" >> $GITHUB_ENV
+
+      - name: Publish release
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        uses: softprops/action-gh-release@v1
+        with:
+          tag_name: ${{ env.TAG_NAME }}
+          name: ${{ env.TAG_NAME }}
+          prerelease: false
+          draft: false
+          files: ${{ env.WHEEL_FILENAME }}
+      - name: Drop outdated wheels
+        uses: dev-drprasad/delete-older-releases@v0.2.0
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        with:
+          keep_latest: 1
+          delete_tag_pattern: wheel- # defaults to ""
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,3 @@
+*.egg-info
+__pycache__
+xemutest/data

Dockerfile

@@ -1,3 +1,12 @@
+#
+# Build test data
+#
+FROM ghcr.io/xboxdev/nxdk AS data
+RUN mkdir /data
+COPY test-xbe /test-xbe
+RUN /usr/src/nxdk/docker_entry.sh make -C /test-xbe
+RUN cp /test-xbe/tester.iso /data
+
 #
 # Build base test container image
 #
@@ -29,51 +38,21 @@ RUN set -xe; \
         zlib1g \
         ;
 
-#
-# Build pyfatx for HDD management
-#
-FROM ubuntu:20.04 AS pyfatx
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update \
- && apt-get install -qy \
-     build-essential \
-     cmake \
-     git \
-     python3-pip
-RUN git clone --depth=1 https://github.com/mborgerson/fatx \
- && mkdir -p /whl \
- && python3 -m pip wheel -w /whl ./fatx
-
-#
-# Build test ISO
-#
-FROM ghcr.io/xboxdev/nxdk AS test-iso-1
-COPY test-xbe /test-xbe
-RUN /usr/src/nxdk/docker_entry.sh make -C /test-xbe
-
 #
 # Build final test container
 #
 FROM run-container-base AS test-container
-
-RUN useradd -ms /bin/bash user
-
-COPY --from=pyfatx /whl /whl
-RUN python3 -m pip install --find-links /whl /whl/pyfatx-*.whl
-
 ENV DEBIAN_FRONTEND=noninteractive
 ENV SDL_AUDIODRIVER=dummy
 
 # VNC port for debugging
 EXPOSE 5900
 
-COPY docker_entry.sh /docker_entry.sh
-ENTRYPOINT ["/docker_entry.sh"]
-
 RUN mkdir /work
-COPY test.py /work/test.py
-COPY xbox_hdd.qcow2 /work/xbox_hdd.qcow2
-COPY --from=test-iso-1 /test-xbe/tester.iso /work/tester.iso
-
 WORKDIR /work
-CMD ["/usr/bin/python3", "/work/test.py"]
+COPY scripts/docker_entry.sh /docker_entry.sh
+COPY . /work/xemu-test
+COPY --from=data /data /work/xemu-test/xemutest/
+RUN pip install /work/xemu-test
+ENTRYPOINT ["/docker_entry.sh"]
+CMD ["/usr/bin/python3", "-m", "xemutest", "/work/private", "/work/results"]

MANIFEST.in

@@ -0,0 +1,2 @@
+graft test-xbe
+graft Dockerfile

scripts/build_test_data.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+if [[ ! -d xemutest ]]; then
+	echo "Run from root dir"
+	exit 1
+fi
+
+set -ex
+target=data
+image=xemu-test-data-tmp-img
+docker build --target $target -t $image .
+container=$(docker create $image "")
+docker cp $container:/data xemutest/
+docker rm $container
+docker rmi $image

setup.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+from setuptools import setup
+
+
+__version__ = '0.0.1'
+
+
+setup(name='xemutest',
+    version=__version__,
+    description='xemu Automated Tests',
+    author='Matt Borgerson',
+    author_email='contact@mborgerson.com',
+    url='https://github.com/mborgerson/xemu-test',
+    packages=['xemutest'],
+    include_package_data=True,
+    package_data={'xemutest': ['data/*']},
+    install_requires=[
+        'pyfatx',
+        'pywinauto; sys_platform == "win32"'
+        ],
+    python_requires='>=3.6'
+    )

test.py

@@ -1,126 +0,0 @@
-#!/usr/bin/env python3
-
-import subprocess
-import shutil
-import logging
-import os
-import signal
-import time
-
-logging.basicConfig(level=logging.INFO)
-_l = logging.getLogger(__file__)
-
-class Test:
-	"""
-	Test provides a basic framework that:
-	- Starts FFMPEG to record footage of xemu while it runs
-	- Launches xemu with an test XBE loaded from a disc image
-	- Waits for xemu to shutdown or timeout
-	- Inspect the filesystem for test results
-	"""
-
-	def __init__(self):
-		self.flash_path         = '/work/private/bios.bin'
-		self.mcpx_path          = '/work/private/mcpx.bin'
-		self.blank_hdd_path     = '/work/xbox_hdd.qcow2'
-		self.hdd_path           = '/tmp/test.img'
-		self.mount_path         = '/tmp/xemu-hdd-mount'
-		self.iso_path           = '/work/tester.iso'
-		self.results_in_path    = os.path.join(self.mount_path, 'results')
-		self.results_out_path   = '/work/results'
-		self.video_capture_path = os.path.join(self.results_out_path, 'capture.mp4')
-		self.timeout            = 60
-
-	def prepare_roms(self):
-		_l.info('Preparing ROM images')
-		# Nothing to do here yet
-
-	def prepare_hdd(self):
-		_l.info('Preparing HDD image')
-		# FIXME: Replace qcow2 with pyfatx disk init
-		subprocess.run(f'qemu-img convert {self.blank_hdd_path} {self.hdd_path}'.split(), check=True)
-
-	def prepare_config(self):
-		config = ('[system]\n'
-                  f'flash_path = {self.flash_path}\n'
-                  f'bootrom_path = {self.mcpx_path}\n'
-                  f'hdd_path = {self.hdd_path}\n'
-                  'shortanim = true\n'
-                  )
-		_l.info('Prepared config file:\n%s', config)
-		with open('xemu.ini', 'w') as f:
-			f.write(config)
-
-	def launch_ffmpeg(self):
-		_l.info('Launching FFMPEG (capturing to %s)', self.video_capture_path)
-		c = ('/usr/bin/ffmpeg -loglevel error '
-			 f'-video_size 640x480 -f x11grab -i {os.getenv("DISPLAY")} '
-			 f'-c:v libx264 -preset fast -profile:v baseline -pix_fmt yuv420p '
-			 f'{self.video_capture_path} -y')
-		self.ffmpeg = subprocess.Popen(c.split())
-
-	def terminate_ffmpeg(self):
-		_l.info('Shutting down FFMPEG')
-		self.ffmpeg.send_signal(signal.SIGINT)
-		for _ in range(10):
-			self.ffmpeg.poll()
-			if self.ffmpeg.returncode is not None:
-				_l.info('FFMPEG exited %d', self.ffmpeg.returncode)
-				break
-			time.sleep(0.1)
-		self.ffmpeg.poll()
-		if self.ffmpeg.returncode is None:
-			_l.warning('Terminating FFMPEG')
-			self.ffmpeg.terminate()
-
-	def launch_xemu(self):
-		_l.info('Launching xemu...')
-		c = (f'timeout {self.timeout} '
-			 f'xemu -config_path ./xemu.ini -dvd_path {self.iso_path} -full-screen')
-		subprocess.run(c.split(), check=True)
-
-	def mount_hdd(self):
-		_l.info('Mounting HDD image')
-		os.makedirs(self.mount_path, exist_ok=True)
-		subprocess.run(f'python3 -m pyfatx -x {self.hdd_path}'.split(), check=True, cwd=self.mount_path)
-
-	def copy_results(self):
-		_l.info('Copying test results...')
-		shutil.copytree(self.results_in_path, self.results_out_path, dirs_exist_ok=True)
-
-	def unmount_hdd(self):
-		_l.info('Unmounting HDD image')
-		# Nothing to do
-
-	def analyze_results(self):
-		with open(os.path.join(self.results_out_path, 'results.txt')) as f:
-			assert(f.read().strip() == 'Success')
-
-	def run(self):
-		os.makedirs(self.results_out_path, exist_ok=True)
-		self.prepare_roms()
-		self.prepare_hdd()
-		self.prepare_config()
-		self.launch_ffmpeg()
-		self.launch_xemu()
-		self.terminate_ffmpeg()
-		self.mount_hdd()
-		self.copy_results()
-		self.unmount_hdd()
-		self.analyze_results()
-
-def main():
-	result = True
-	tests = [Test]
-	for test_cls in tests:
-		try:
-			test_cls().run()
-			print('Test passed!')
-		except:
-			_l.exception('Test failed!')
-			result = False
-
-	exit(0 if result else 1)
-
-if __name__ == '__main__':
-	main()

xemutest/__init__.py

@@ -0,0 +1 @@
+from .test import Test

xemutest/__main__.py

@@ -0,0 +1,32 @@
+import logging
+import argparse
+
+from xemutest import Test
+
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger(__file__)
+
+
+def main():
+	ap = argparse.ArgumentParser()
+	ap.add_argument('private', help='Path to private data files')
+	ap.add_argument('results', help='Path to directory where results should go')
+	args = ap.parse_args()
+
+	result = True
+	tests = [Test]
+	for i, test_cls in enumerate(tests):
+		log.info('Test %d', i)
+		log.info('-'*40)
+		try:
+			test_cls(args.private, args.results).run()
+			log.info('Test %d passed!', i)
+		except:
+			log.exception('Test %d failed!', i)
+			result = False
+
+	exit(0 if result else 1)
+
+if __name__ == '__main__':
+	main()

xemutest/test.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+import subprocess
+import shutil
+import logging
+import os
+import signal
+import time
+import platform
+import sys
+from typing import Optional
+
+from pyfatx import Fatx
+
+
+if platform.system() == 'Windows':
+	import pywinauto.application
+
+
+log = logging.getLogger(__file__)
+
+
+class Test:
+	"""
+	Test provides a basic framework that:
+	- Starts FFMPEG to record footage of xemu while it runs
+	- Launches xemu with an test XBE loaded from a disc image
+	- Waits for xemu to shutdown or timeout
+	- Inspect the filesystem for test results
+
+	Tester runs in current working directory and will generate some working files.
+	"""
+
+	def __init__(self, private_path: str, results_path: str):
+		cur_dir = os.getcwd()
+		if platform.system() == 'Windows':
+			self.xemu_path = os.path.join(cur_dir, 'xemu.exe')
+		else:
+			self.xemu_path = 'xemu'
+
+		test_data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data'))
+		if not os.path.isdir(test_data_path):
+			raise FileNotFoundError('Test data was not installed with the package. You need to build it.')
+
+		self.flash_path         = os.path.join(private_path, 'bios.bin')
+		self.mcpx_path          = os.path.join(private_path, 'mcpx.bin')
+		self.hdd_path           = os.path.join(cur_dir, 'test.img')
+		self.mount_path         = os.path.join(cur_dir, 'xemu-hdd-mount')
+		self.iso_path           = os.path.join(test_data_path, 'tester.iso')
+		self.results_in_path    = os.path.join(self.mount_path, 'results')
+		self.results_out_path   = results_path
+		self.video_capture_path = os.path.join(self.results_out_path, 'capture.mp4')
+		self.timeout            = 60
+
+		if platform.system() == 'Windows':
+			self.app: Optional[pywinauto.application.Application] = None
+			self.record_x: int = 0
+			self.record_y: int = 0
+			self.record_w: int = 0
+			self.record_h: int = 0
+
+	def prepare_roms(self):
+		log.info('Preparing ROM images')
+		# Nothing to do here yet
+
+	def prepare_hdd(self):
+		log.info('Preparing HDD image')
+		disk_size = 8*1024*1024*1024
+		if os.path.exists(self.hdd_path):
+			if os.path.getsize(self.hdd_path) != disk_size:
+				raise FileExistsError('Target image path exists and is not expected size')
+			Fatx.format(self.hdd_path)
+		else:
+			Fatx.create(self.hdd_path, disk_size)
+
+	def prepare_config(self):
+		config = ( '[system]\n'
+                  f'flash_path = {self.flash_path}\n'
+                  f'bootrom_path = {self.mcpx_path}\n'
+                  f'hdd_path = {self.hdd_path}\n'
+                   'shortanim = true\n'
+                   '[misc]\n'
+                   'check_for_update = false\n'
+                  )
+		log.info('Prepared config file:\n%s', config)
+		with open('xemu.ini', 'w') as f:
+			f.write(config)
+
+	def launch_video_capture(self):
+		log.info('Launching FFMPEG (capturing to %s)', self.video_capture_path)
+		if platform.system() == 'Windows':
+			# FIXME: Assuming nvenc available on host. Add check for this.
+			c = ['ffmpeg.exe', '-loglevel', 'error', '-framerate', '60',
+				'-video_size', f'{self.record_w}x{self.record_h}', '-f', 'gdigrab', '-offset_x', f'{self.record_x}', '-offset_y', f'{self.record_y}', '-i', 'desktop',
+				'-c:v', 'h264_nvenc', '-pix_fmt', 'yuv420p',
+				self.video_capture_path, '-y']
+		else:
+			c = ['ffmpeg', '-loglevel', 'error',
+				 '-video_size', '640x480', '-f', 'x11grab', '-i', os.getenv("DISPLAY"),
+				 '-c:v', 'libx264', '-preset', 'fast', '-profile:v', 'baseline', '-pix_fmt', 'yuv420p',
+				 self.video_capture_path, '-y']
+		self.ffmpeg = subprocess.Popen(c, stdin=subprocess.PIPE)
+
+	def terminate_video_capture(self):
+		log.info('Shutting down FFMPEG')
+		self.ffmpeg.communicate(b'q\n', timeout=5)
+
+	def launch_xemu(self):
+		log.info('Launching xemu...')
+
+		if platform.system() == 'Windows':
+			c = [self.xemu_path, '-config_path', './xemu.ini', '-dvd_path', self.iso_path]
+		else:
+			c = [self.xemu_path, '-config_path', './xemu.ini', '-dvd_path', self.iso_path, '-full-screen']
+		start = time.time()
+		xemu = subprocess.Popen(c)
+
+		if platform.system() == 'Windows':
+			self.app = pywinauto.application.Application()
+			self.app.connect(process=xemu.pid)
+			main_window = self.app.window(title_re=r'^xemu \| v.+')
+			if main_window is None:
+				raise Exception('Failed to find main xemu window...')
+
+			target_width = 640
+			target_height = 480
+
+			rect = main_window.client_area_rect()
+			cx, cy, cw, ch = rect.left, rect.top, rect.width(), rect.height()
+			rect = main_window.rectangle()
+			x, y, w, h = rect.left, rect.top, rect.width(), rect.height()
+
+			main_window.move_window(0, 0,
+				                    target_width + (w-cw),
+				                    target_height + (h-ch))
+			rect = main_window.client_area_rect()
+			x, y, w, h = rect.left, rect.top, rect.width(), rect.height()
+			log.info('xemu window is at %d,%d w=%d,h=%d', x, y, w, h)
+			self.record_x = x
+			self.record_y = y
+			self.record_w = w
+			self.record_h = h
+
+		self.launch_video_capture()
+
+		while True:
+			status = xemu.poll()
+			if status is not None:
+				log.info('xemu exited %d', status)
+				break
+			now = time.time()
+			if (now - start) > self.timeout:
+				log.info('Timeout exceeded. Terminating.')
+				xemu.kill()
+				xemu.wait()
+				break
+			time.sleep(1)
+
+		self.terminate_video_capture()
+
+	def mount_hdd(self):
+		log.info('Mounting HDD image')
+		if os.path.exists(self.mount_path):
+			shutil.rmtree(self.mount_path)
+		os.makedirs(self.mount_path, exist_ok=True)
+
+		# FIXME: Don't need to run here
+		subprocess.run([sys.executable, '-m', 'pyfatx', '-x', self.hdd_path], check=True, cwd=self.mount_path)
+
+	def copy_results(self):
+		log.info('Copying test results...')
+		shutil.copytree(self.results_in_path, self.results_out_path, dirs_exist_ok=True)
+
+	def unmount_hdd(self):
+		log.info('Unmounting HDD image')
+		# Nothing to do
+
+	def analyze_results(self):
+		with open(os.path.join(self.results_out_path, 'results.txt')) as f:
+			assert(f.read().strip() == 'Success')
+
+	def run(self):
+		os.makedirs(self.results_out_path, exist_ok=True)
+		self.prepare_roms()
+		self.prepare_hdd()
+		self.prepare_config()
+		self.launch_xemu()
+		self.mount_hdd()
+		self.copy_results()
+		self.unmount_hdd()
+		self.analyze_results()