Reorganize

.gitignore

@@ -1,5 +0,0 @@
-results
-bios.bin
-mcpx.bin
-xemu.ini
-xemu.deb

Dockerfile

@@ -0,0 +1,79 @@
+#
+# Build base test container image
+#
+FROM ubuntu:20.04 as run-container-base
+ENV DEBIAN_FRONTEND=noninteractive
+RUN set -xe; \
+    apt-get -qy update \
+    && apt-get -qy install \
+        python3-pip \
+        xvfb \
+        x11-utils \
+        x11vnc \
+        xinit \
+        ffmpeg \
+        i3 \
+        qemu-utils \
+        libc6 \
+        libepoxy0 \
+        libgcc-s1 \
+        libglib2.0-0 \
+        libgtk-3-0 \
+        libpcap0.8 \
+        libpixman-1-0 \
+        libpulse0 \
+        libsamplerate0 \
+        libsdl2-2.0-0 \
+        libssl1.1 \
+        libstdc++6 \
+        zlib1g \
+        ;
+
+#
+# Build pyfatx for HDD management
+#
+FROM ubuntu:20.04 AS pyfatx
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+ && apt-get install -qy \
+     build-essential \
+     cmake \
+     git \
+     python3-pip
+RUN git clone --depth=1 https://github.com/mborgerson/fatx \
+ && mkdir -p /whl \
+ && python3 -m pip wheel -w /whl ./fatx
+
+#
+# Build test ISO
+#
+FROM ghcr.io/xboxdev/nxdk AS test-iso-1
+COPY test-xbe /test-xbe
+RUN /usr/src/nxdk/docker_entry.sh make -C /test-xbe
+
+#
+# Build final test container
+#
+FROM run-container-base
+
+RUN useradd -ms /bin/bash user
+
+COPY --from=pyfatx /whl /whl
+RUN python3 -m pip install --find-links /whl /whl/pyfatx-*.whl
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV SDL_AUDIODRIVER=dummy
+
+# VNC port for debugging
+EXPOSE 5900
+
+COPY docker_entry.sh /docker_entry.sh
+ENTRYPOINT ["/docker_entry.sh"]
+
+RUN mkdir /work
+COPY test.py /work/test.py
+COPY xbox_hdd.qcow2 /work/xbox_hdd.qcow2
+COPY --from=test-iso-1 /test-xbe/tester.iso /work/tester.iso
+
+WORKDIR /work
+CMD ["/usr/bin/python3", "/work/test.py"]

README.md

@@ -0,0 +1,43 @@
+xemu Automated Testing
+======================
+
+Performs a suite of tests against a build of xemu, capturing test results and
+footage of the runs. Primarily used for CI testing of xemu.
+
+Containerized Testing
+---------------------
+This testing system is intended to be able to be run inside a container for
+reproducability and for regular testing on cheap cloud VMs.
+
+To build the container image:
+
+	docker build -t xemu-test .
+
+This repository also has GitHub actions set up to automatically build and
+publish the container image to the GitHub container registry, so you can pull
+that image for testing:
+
+	docker pull ghcr.io/mborgerson/xemu-test:master
+
+Set up the following dir structure:
+
+- /work/results: Results will be copied here
+- /work/private: Directory for ROMs and other files
+  - /work/private/mcpx.bin
+  - /work/private/bios.bin
+- /work/inputs: Directory containing xemu build to test
+  - /work/xemu.deb
+
+Then run with something like:
+
+	docker run --rm -it \
+		-v $PWD/results:/work/results \
+		-v $PWD/private:/work/private \
+		-v $PWD/inputs:/work/inputs \
+		-p 5900:5900 \
+		ghcr.io/mborgerson/xemu-test:master
+
+xemu is running headless when in the container, so if you need to interact with
+it you can connect to the container VNC server with:
+
+	xtightvncviewer 127.0.0.1

docker_entry.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+exec 2>&1
+
 XVFB_WHD=640x480x24
 DISPLAY=:99
 
@@ -9,15 +11,18 @@ fi
 
 set -e
 echo "[*] Installing xemu package"
-apt-get -qy install /work/xemu.deb
+apt-get -qy install /work/inputs/xemu.deb
 
 echo "exec i3" >> ~/.xinitrc
 chmod +x ~/.xinitrc
+
 mkdir -p ~/.config/i3
-echo "border none" >> ~/.config/i3/config
+cat <<EOF >>~/.config/i3/config
+border none
+EOF
 
 echo "[*] Starting Xvfb"
-xinit -- /usr/bin/Xvfb $DISPLAY -ac -screen 0 "$XVFB_WHD" -nolisten tcp +extension GLX +render -noreset &
+xinit -- /usr/bin/Xvfb $DISPLAY -ac -screen 0 "$XVFB_WHD" -nolisten tcp +extension GLX +render -noreset & 1>/dev/null 2>&1 &
 Xvfb_pid="$!"
 echo "[~] Waiting for Xvfb (PID: $Xvfb_pid) to be ready..."
 set +e

test-container/Dockerfile

@@ -1,48 +0,0 @@
-FROM ubuntu:20.04
-RUN apt-get update \
- && DEBIAN_FRONTEND=noninteractive \
-    apt-get install -qy \
-        build-essential cmake git python3-pip
-RUN git clone --depth=1 https://github.com/mborgerson/fatx \
- && cd fatx \
- && mkdir -p /whl \
- && python3 -m pip wheel -w /whl .
-
-FROM ubuntu:20.04
-RUN set -xe; \
-    apt-get -qy update \
-    && DEBIAN_FRONTEND=noninteractive \
-    apt-get -qy install \
-        python3-pip \
-        xvfb \
-        x11-utils \
-        x11vnc \
-        xinit \
-        ffmpeg \
-        i3 \
-        qemu-utils \
-        libc6 \
-        libepoxy0 \
-        libgcc-s1 \
-        libglib2.0-0 \
-        libgtk-3-0 \
-        libpcap0.8 \
-        libpixman-1-0 \
-        libpulse0 \
-        libsamplerate0 \
-        libsdl2-2.0-0 \
-        libssl1.1 \
-        libstdc++6 \
-        zlib1g \
-        cpu-checker \
-        ;
-
-COPY --from=0 /whl /whl
-RUN python3 -m pip install --find-links /whl /whl/pyfatx-*.whl
-
-ENV SDL_AUDIODRIVER=disk
-ENV SDL_DISKAUDIOFILE=/dev/null
-EXPOSE 5900
-
-COPY ./docker_entry.sh /usr/local/bin/docker_entry.sh
-ENTRYPOINT ["/usr/local/bin/docker_entry.sh"]

test-xbe/.gitignore

@@ -1,5 +0,0 @@
-bin
-*.d
-*.exe
-*.obj
-*.iso

test-xbe/main.c

@@ -1,4 +1,5 @@
 #include <hal/debug.h>
+
 #include <hal/video.h>
 #include <windows.h>
 #include <nxdk/mount.h>
@@ -9,7 +10,10 @@ int main(void)
 {
     XVideoSetMode(640, 480, 32, REFRESH_DEFAULT);
 
-    debugPrint("Hello nxdk!\n");
+    for (int i = 0; i < 10; i++) {
+        debugPrint("Hello nxdk!\n");
+        Sleep(500);
+    }
 
     BOOL ret = nxMountDrive('C', "\\Device\\Harddisk0\\Partition2\\");
     if (!ret) {
@@ -28,6 +32,7 @@ int main(void)
     fwrite(buf, strlen(buf), 1, f);
     fclose(f);
 
+
 shutdown:
     HalInitiateShutdown();
     while (1) {

test.py

@@ -11,13 +11,21 @@ logging.basicConfig(level=logging.INFO)
 _l = logging.getLogger(__file__)
 
 class Test:
+	"""
+	Test provides a basic framework that:
+	- Starts FFMPEG to record footage of xemu while it runs
+	- Launches xemu with an test XBE loaded from a disc image
+	- Waits for xemu to shutdown or timeout
+	- Inspect the filesystem for test results
+	"""
+
 	def __init__(self):
 		self.flash_path         = '/work/private/bios.bin'
 		self.mcpx_path          = '/work/private/mcpx.bin'
 		self.blank_hdd_path     = '/work/xbox_hdd.qcow2'
 		self.hdd_path           = '/tmp/test.img'
 		self.mount_path         = '/tmp/xemu-hdd-mount'
-		self.iso_path           = '/work/test-xbe/tester.iso'
+		self.iso_path           = '/work/tester.iso'
 		self.results_in_path    = os.path.join(self.mount_path, 'results')
 		self.results_out_path   = '/work/results'
 		self.video_capture_path = os.path.join(self.results_out_path, 'capture.mp4')
@@ -25,10 +33,11 @@ class Test:
 
 	def prepare_roms(self):
 		_l.info('Preparing ROM images')
-		# TODO
+		# Nothing to do here yet
 
 	def prepare_hdd(self):
 		_l.info('Preparing HDD image')
+		# FIXME: Replace qcow2 with pyfatx disk init
 		subprocess.run(f'qemu-img convert {self.blank_hdd_path} {self.hdd_path}'.split(), check=True)
 
 	def prepare_config(self):
@@ -67,9 +76,8 @@ class Test:
 	def launch_xemu(self):
 		_l.info('Launching xemu...')
 		c = (f'timeout {self.timeout} '
-			 f'xemu -config_path ./xemu.ini -dvd_path {self.iso_path} '
-			 '-full-screen')
-		subprocess.run(c.split())
+			 f'xemu -config_path ./xemu.ini -dvd_path {self.iso_path} -full-screen')
+		subprocess.run(c.split(), check=True)
 
 	def mount_hdd(self):
 		_l.info('Mounting HDD image')
@@ -102,8 +110,17 @@ class Test:
 		self.analyze_results()
 
 def main():
-	test = Test()
-	test.run()
+	result = True
+	tests = [Test]
+	for test_cls in tests:
+		try:
+			test_cls().run()
+			print('Test passed!')
+		except:
+			_l.exception('Test failed!')
+			result = False
+
+	exit(0 if result else 1)
 
 if __name__ == '__main__':
 	main()

test.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-set -e
-set -o pipefail
-
-SUPPORT_DIR=private
-SUPPORT_ARCHIVE=${SUPPORT_DIR}.zip
-SUPPORT_ARCHIVE_ENC=${SUPPORT_ARCHIVE}.enc
-SUPPORT_URL=${SUPPORT_URL:-http://localhost:8080/${SUPPORT_ARCHIVE_ENC}}
-
-if [[ ! -d ${SUPPORT_DIR} ]]; then
-    if [[ ! -e ${SUPPORT_ARCHIVE} ]]; then
-        if [[ ! -e ${SUPPORT_ARCHIVE_ENC} ]]; then
-            echo "[*] Downloading ${SUPPORT_ARCHIVE_ENC}"
-            wget -O ${SUPPORT_ARCHIVE_ENC} ${SUPPORT_URL} 1>/dev/null 2>&1
-        fi
-        echo "[*] Decrypting ${SUPPORT_ARCHIVE}"
-        gpg --quiet --batch --yes --decrypt --passphrase="$SUPPORT_PASSPHRASE" \
-            --output ./${SUPPORT_ARCHIVE} ${SUPPORT_ARCHIVE_ENC}
-    fi
-
-    unzip ${SUPPORT_ARCHIVE}
-fi
-
-echo "[*] Building test executable"
-docker run --rm -v $PWD/test-xbe:/work -w /work ghcr.io/xboxdev/nxdk make
-
-echo "[*] Pulling test container"
-docker pull ghcr.io/mborgerson/xemu-test:master
-
-echo "[*] Running tests"
-rm -rf results
-mkdir results
-docker run --rm -p 5900:5900 -v $PWD:/work -w /work \
-           ghcr.io/mborgerson/xemu-test:master \
-           python3 test_main.py 2>&1 | tee results/log.txt