diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 62064c5..0000000 --- a/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -results -bios.bin -mcpx.bin -xemu.ini -xemu.deb diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..718a390 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,79 @@ +# +# Build base test container image +# +FROM ubuntu:20.04 as run-container-base +ENV DEBIAN_FRONTEND=noninteractive +RUN set -xe; \ + apt-get -qy update \ + && apt-get -qy install \ + python3-pip \ + xvfb \ + x11-utils \ + x11vnc \ + xinit \ + ffmpeg \ + i3 \ + qemu-utils \ + libc6 \ + libepoxy0 \ + libgcc-s1 \ + libglib2.0-0 \ + libgtk-3-0 \ + libpcap0.8 \ + libpixman-1-0 \ + libpulse0 \ + libsamplerate0 \ + libsdl2-2.0-0 \ + libssl1.1 \ + libstdc++6 \ + zlib1g \ + ; + +# +# Build pyfatx for HDD management +# +FROM ubuntu:20.04 AS pyfatx +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -qy \ + build-essential \ + cmake \ + git \ + python3-pip +RUN git clone --depth=1 https://github.com/mborgerson/fatx \ + && mkdir -p /whl \ + && python3 -m pip wheel -w /whl ./fatx + +# +# Build test ISO +# +FROM ghcr.io/xboxdev/nxdk AS test-iso-1 +COPY test-xbe /test-xbe +RUN /usr/src/nxdk/docker_entry.sh make -C /test-xbe + +# +# Build final test container +# +FROM run-container-base + +RUN useradd -ms /bin/bash user + +COPY --from=pyfatx /whl /whl +RUN python3 -m pip install --find-links /whl /whl/pyfatx-*.whl + +ENV DEBIAN_FRONTEND=noninteractive +ENV SDL_AUDIODRIVER=dummy + +# VNC port for debugging +EXPOSE 5900 + +COPY docker_entry.sh /docker_entry.sh +ENTRYPOINT ["/docker_entry.sh"] + +RUN mkdir /work +COPY test.py /work/test.py +COPY xbox_hdd.qcow2 /work/xbox_hdd.qcow2 +COPY --from=test-iso-1 /test-xbe/tester.iso /work/tester.iso + +WORKDIR /work +CMD ["/usr/bin/python3", "/work/test.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..9942567 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +xemu Automated Testing +====================== + +Performs a suite of tests against a build of xemu, capturing test results and +footage of the runs. Primarily used for CI testing of xemu. + +Containerized Testing +--------------------- +This testing system is intended to be able to be run inside a container for +reproducability and for regular testing on cheap cloud VMs. + +To build the container image: + + docker build -t xemu-test . + +This repository also has GitHub actions set up to automatically build and +publish the container image to the GitHub container registry, so you can pull +that image for testing: + + docker pull ghcr.io/mborgerson/xemu-test:master + +Set up the following dir structure: + +- /work/results: Results will be copied here +- /work/private: Directory for ROMs and other files + - /work/private/mcpx.bin + - /work/private/bios.bin +- /work/inputs: Directory containing xemu build to test + - /work/xemu.deb + +Then run with something like: + + docker run --rm -it \ + -v $PWD/results:/work/results \ + -v $PWD/private:/work/private \ + -v $PWD/inputs:/work/inputs \ + -p 5900:5900 \ + ghcr.io/mborgerson/xemu-test:master + +xemu is running headless when in the container, so if you need to interact with +it you can connect to the container VNC server with: + + xtightvncviewer 127.0.0.1 diff --git a/test-container/docker_entry.sh b/docker_entry.sh similarity index 78% rename from test-container/docker_entry.sh rename to docker_entry.sh index f51f86f..b1a89c0 100755 --- a/test-container/docker_entry.sh +++ b/docker_entry.sh @@ -1,4 +1,6 @@ #!/bin/bash +exec 2>&1 + XVFB_WHD=640x480x24 DISPLAY=:99 @@ -9,15 +11,18 @@ fi set -e echo "[*] Installing xemu package" -apt-get -qy install /work/xemu.deb +apt-get -qy install /work/inputs/xemu.deb echo "exec i3" >> ~/.xinitrc chmod +x ~/.xinitrc + mkdir -p ~/.config/i3 -echo "border none" >> ~/.config/i3/config +cat <>~/.config/i3/config +border none +EOF echo "[*] Starting Xvfb" -xinit -- /usr/bin/Xvfb $DISPLAY -ac -screen 0 "$XVFB_WHD" -nolisten tcp +extension GLX +render -noreset & +xinit -- /usr/bin/Xvfb $DISPLAY -ac -screen 0 "$XVFB_WHD" -nolisten tcp +extension GLX +render -noreset & 1>/dev/null 2>&1 & Xvfb_pid="$!" echo "[~] Waiting for Xvfb (PID: $Xvfb_pid) to be ready..." set +e diff --git a/test-container/Dockerfile b/test-container/Dockerfile deleted file mode 100644 index c871c5d..0000000 --- a/test-container/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM ubuntu:20.04 -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive \ - apt-get install -qy \ - build-essential cmake git python3-pip -RUN git clone --depth=1 https://github.com/mborgerson/fatx \ - && cd fatx \ - && mkdir -p /whl \ - && python3 -m pip wheel -w /whl . - -FROM ubuntu:20.04 -RUN set -xe; \ - apt-get -qy update \ - && DEBIAN_FRONTEND=noninteractive \ - apt-get -qy install \ - python3-pip \ - xvfb \ - x11-utils \ - x11vnc \ - xinit \ - ffmpeg \ - i3 \ - qemu-utils \ - libc6 \ - libepoxy0 \ - libgcc-s1 \ - libglib2.0-0 \ - libgtk-3-0 \ - libpcap0.8 \ - libpixman-1-0 \ - libpulse0 \ - libsamplerate0 \ - libsdl2-2.0-0 \ - libssl1.1 \ - libstdc++6 \ - zlib1g \ - cpu-checker \ - ; - -COPY --from=0 /whl /whl -RUN python3 -m pip install --find-links /whl /whl/pyfatx-*.whl - -ENV SDL_AUDIODRIVER=disk -ENV SDL_DISKAUDIOFILE=/dev/null -EXPOSE 5900 - -COPY ./docker_entry.sh /usr/local/bin/docker_entry.sh -ENTRYPOINT ["/usr/local/bin/docker_entry.sh"] diff --git a/test-xbe/.gitignore b/test-xbe/.gitignore deleted file mode 100644 index eafc6b3..0000000 --- a/test-xbe/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -bin -*.d -*.exe -*.obj -*.iso diff --git a/test-xbe/main.c b/test-xbe/main.c index 6fece94..76f93a7 100644 --- a/test-xbe/main.c +++ b/test-xbe/main.c @@ -1,4 +1,5 @@ #include + #include #include #include @@ -9,7 +10,10 @@ int main(void) { XVideoSetMode(640, 480, 32, REFRESH_DEFAULT); - debugPrint("Hello nxdk!\n"); + for (int i = 0; i < 10; i++) { + debugPrint("Hello nxdk!\n"); + Sleep(500); + } BOOL ret = nxMountDrive('C', "\\Device\\Harddisk0\\Partition2\\"); if (!ret) { @@ -28,6 +32,7 @@ int main(void) fwrite(buf, strlen(buf), 1, f); fclose(f); + shutdown: HalInitiateShutdown(); while (1) { diff --git a/test_main.py b/test.py old mode 100644 new mode 100755 similarity index 81% rename from test_main.py rename to test.py index 5a18070..05f118c --- a/test_main.py +++ b/test.py @@ -11,13 +11,21 @@ logging.basicConfig(level=logging.INFO) _l = logging.getLogger(__file__) class Test: + """ + Test provides a basic framework that: + - Starts FFMPEG to record footage of xemu while it runs + - Launches xemu with an test XBE loaded from a disc image + - Waits for xemu to shutdown or timeout + - Inspect the filesystem for test results + """ + def __init__(self): self.flash_path = '/work/private/bios.bin' self.mcpx_path = '/work/private/mcpx.bin' self.blank_hdd_path = '/work/xbox_hdd.qcow2' self.hdd_path = '/tmp/test.img' self.mount_path = '/tmp/xemu-hdd-mount' - self.iso_path = '/work/test-xbe/tester.iso' + self.iso_path = '/work/tester.iso' self.results_in_path = os.path.join(self.mount_path, 'results') self.results_out_path = '/work/results' self.video_capture_path = os.path.join(self.results_out_path, 'capture.mp4') @@ -25,10 +33,11 @@ class Test: def prepare_roms(self): _l.info('Preparing ROM images') - # TODO + # Nothing to do here yet def prepare_hdd(self): _l.info('Preparing HDD image') + # FIXME: Replace qcow2 with pyfatx disk init subprocess.run(f'qemu-img convert {self.blank_hdd_path} {self.hdd_path}'.split(), check=True) def prepare_config(self): @@ -67,9 +76,8 @@ class Test: def launch_xemu(self): _l.info('Launching xemu...') c = (f'timeout {self.timeout} ' - f'xemu -config_path ./xemu.ini -dvd_path {self.iso_path} ' - '-full-screen') - subprocess.run(c.split()) + f'xemu -config_path ./xemu.ini -dvd_path {self.iso_path} -full-screen') + subprocess.run(c.split(), check=True) def mount_hdd(self): _l.info('Mounting HDD image') @@ -102,8 +110,17 @@ class Test: self.analyze_results() def main(): - test = Test() - test.run() + result = True + tests = [Test] + for test_cls in tests: + try: + test_cls().run() + print('Test passed!') + except: + _l.exception('Test failed!') + result = False + + exit(0 if result else 1) if __name__ == '__main__': main() diff --git a/test.sh b/test.sh deleted file mode 100755 index 09b31a2..0000000 --- a/test.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -SUPPORT_DIR=private -SUPPORT_ARCHIVE=${SUPPORT_DIR}.zip -SUPPORT_ARCHIVE_ENC=${SUPPORT_ARCHIVE}.enc -SUPPORT_URL=${SUPPORT_URL:-http://localhost:8080/${SUPPORT_ARCHIVE_ENC}} - -if [[ ! -d ${SUPPORT_DIR} ]]; then - if [[ ! -e ${SUPPORT_ARCHIVE} ]]; then - if [[ ! -e ${SUPPORT_ARCHIVE_ENC} ]]; then - echo "[*] Downloading ${SUPPORT_ARCHIVE_ENC}" - wget -O ${SUPPORT_ARCHIVE_ENC} ${SUPPORT_URL} 1>/dev/null 2>&1 - fi - echo "[*] Decrypting ${SUPPORT_ARCHIVE}" - gpg --quiet --batch --yes --decrypt --passphrase="$SUPPORT_PASSPHRASE" \ - --output ./${SUPPORT_ARCHIVE} ${SUPPORT_ARCHIVE_ENC} - fi - - unzip ${SUPPORT_ARCHIVE} -fi - -echo "[*] Building test executable" -docker run --rm -v $PWD/test-xbe:/work -w /work ghcr.io/xboxdev/nxdk make - -echo "[*] Pulling test container" -docker pull ghcr.io/mborgerson/xemu-test:master - -echo "[*] Running tests" -rm -rf results -mkdir results -docker run --rm -p 5900:5900 -v $PWD:/work -w /work \ - ghcr.io/mborgerson/xemu-test:master \ - python3 test_main.py 2>&1 | tee results/log.txt