mirror of
https://github.com/hrydgard/ppsspp.git
synced 2024-11-25 06:19:51 +00:00
dba04a7e22
Needed sometimes to make timing match better for tight loops. Also removes the Puyo Puyo Fever hack.
340 lines
9.2 KiB
Python
Executable File
340 lines
9.2 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Automated script to run the pspautotests test suite in PPSSPP.
|
|
|
|
import sys
|
|
import io
|
|
import os
|
|
import subprocess
|
|
import threading
|
|
|
|
|
|
PPSSPP_EXECUTABLES = [ "Windows\\Release\\PPSSPPHeadless.exe", "build/PPSSPPHeadless" ]
|
|
PPSSPP_EXE = None
|
|
TEST_ROOT = "pspautotests/tests/"
|
|
teamcity_mode = False
|
|
TIMEOUT = 5
|
|
|
|
class Command(object):
|
|
def __init__(self, cmd):
|
|
self.cmd = cmd
|
|
self.process = None
|
|
self.output = None
|
|
self.timeout = False
|
|
|
|
def run(self, timeout):
|
|
def target():
|
|
self.process = subprocess.Popen(self.cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
|
|
self.output, _ = self.process.communicate()
|
|
self.output = self.output.decode("utf-8")
|
|
|
|
thread = threading.Thread(target=target)
|
|
thread.start()
|
|
|
|
thread.join(timeout)
|
|
if thread.is_alive():
|
|
self.timeout = True
|
|
self.process.terminate()
|
|
thread.join()
|
|
|
|
# Test names are the C files without the .c extension.
|
|
# These have worked and should keep working always - regression tests.
|
|
tests_good = [
|
|
"cpu/cpu_alu/cpu_alu",
|
|
"cpu/vfpu/base/vfpu",
|
|
"cpu/vfpu/convert/vfpu_convert",
|
|
"cpu/vfpu/prefixes/vfpu_prefixes",
|
|
"cpu/vfpu/colors/vfpu_colors",
|
|
"cpu/icache/icache",
|
|
"cpu/lsu/lsu",
|
|
"cpu/fpu/fpu",
|
|
|
|
"ctrl/ctrl",
|
|
"ctrl/idle/idle",
|
|
"ctrl/sampling/sampling",
|
|
"ctrl/sampling2/sampling2",
|
|
"display/display",
|
|
"display/vblankmulti",
|
|
"dmac/dmactest",
|
|
"loader/bss/bss",
|
|
"intr/intr",
|
|
"intr/vblank/vblank",
|
|
"misc/testgp",
|
|
"misc/libc",
|
|
"misc/dcache",
|
|
"mstick/mstick",
|
|
"string/string",
|
|
"gpu/callbacks/ge_callbacks",
|
|
"threads/alarm/alarm",
|
|
"threads/alarm/cancel/cancel",
|
|
"threads/alarm/refer/refer",
|
|
"threads/alarm/set/set",
|
|
"threads/callbacks/callbacks",
|
|
"threads/events/events",
|
|
"threads/events/cancel/cancel",
|
|
"threads/events/clear/clear",
|
|
"threads/events/create/create",
|
|
"threads/events/delete/delete",
|
|
"threads/events/poll/poll",
|
|
"threads/events/refer/refer",
|
|
"threads/events/set/set",
|
|
"threads/events/wait/wait",
|
|
"threads/k0/k0",
|
|
"threads/lwmutex/create",
|
|
"threads/lwmutex/delete",
|
|
"threads/lwmutex/lock",
|
|
"threads/lwmutex/priority",
|
|
"threads/lwmutex/refer",
|
|
"threads/lwmutex/try",
|
|
"threads/lwmutex/try600",
|
|
"threads/lwmutex/unlock",
|
|
"threads/mbx/mbx",
|
|
"threads/mbx/cancel/cancel",
|
|
"threads/mbx/create/create",
|
|
"threads/mbx/delete/delete",
|
|
"threads/mbx/poll/poll",
|
|
"threads/mbx/priority/priority",
|
|
"threads/mbx/receive/receive",
|
|
"threads/mbx/refer/refer",
|
|
"threads/mbx/send/send",
|
|
"threads/mutex/create",
|
|
"threads/mutex/delete",
|
|
"threads/mutex/lock",
|
|
"threads/mutex/mutex",
|
|
"threads/mutex/priority",
|
|
"threads/mutex/refer",
|
|
"threads/mutex/try",
|
|
"threads/mutex/unlock",
|
|
"threads/semaphores/semaphores",
|
|
"threads/semaphores/cancel/cancel",
|
|
"threads/semaphores/create/create",
|
|
"threads/semaphores/delete/delete",
|
|
"threads/semaphores/poll/poll",
|
|
"threads/semaphores/priority/priority",
|
|
"threads/semaphores/refer/refer",
|
|
"threads/semaphores/signal/signal",
|
|
"threads/semaphores/wait/wait",
|
|
"threads/threads/release",
|
|
"threads/threads/rotate",
|
|
"threads/threads/threadend",
|
|
"threads/threads/threads",
|
|
"threads/vpl/cancel",
|
|
"threads/vpl/delete",
|
|
"threads/vpl/free",
|
|
"threads/vpl/priority",
|
|
"threads/vpl/refer",
|
|
"threads/vpl/try",
|
|
"threads/vpl/vpl",
|
|
"power/power",
|
|
"umd/callbacks/umd",
|
|
"umd/wait/wait",
|
|
"io/directory/directory",
|
|
]
|
|
|
|
tests_next = [
|
|
# These are the next tests up for fixing. These run by default.
|
|
"audio/atrac/atractest",
|
|
"audio/mp3/mp3test",
|
|
"audio/sascore/sascore",
|
|
"malloc/malloc",
|
|
"threads/fpl/fpl",
|
|
"threads/k0/k0",
|
|
"threads/msgpipe/msgpipe",
|
|
"threads/scheduling/scheduling",
|
|
"threads/threads/create",
|
|
"threads/vtimers/vtimer",
|
|
"threads/vpl/allocate",
|
|
"threads/vpl/create",
|
|
"threads/wakeup/wakeup",
|
|
"gpu/commands/basic",
|
|
"gpu/complex/complex",
|
|
"gpu/displaylist/state",
|
|
"gpu/reflection/reflection",
|
|
"gpu/rendertarget/rendertarget",
|
|
"gpu/signals/jumps",
|
|
"gpu/signals/simple",
|
|
"gpu/simple/simple",
|
|
"gpu/triangle/triangle",
|
|
"hle/check_not_used_uids",
|
|
"font/fonttest",
|
|
"io/cwd/cwd",
|
|
"io/file/file",
|
|
"io/io/io",
|
|
"io/iodrv/iodrv",
|
|
"modules/loadexec/loader",
|
|
"rtc/rtc",
|
|
"sysmem/partition",
|
|
"sysmem/sysmem",
|
|
"umd/io/umd_io",
|
|
"umd/raw_access/raw_access",
|
|
"utility/systemparam/systemparam",
|
|
"video/pmf/pmf",
|
|
"video/pmf_simple/pmf_simple",
|
|
]
|
|
|
|
# These don't even run (or run correctly) on the real PSP
|
|
test_broken = [
|
|
]
|
|
|
|
|
|
# These are the tests we ignore (not important, or impossible to run)
|
|
tests_ignored = [
|
|
"kirk/kirk",
|
|
"me/me",
|
|
|
|
"umd/umd", # mostly fixed but output seems broken? (first retval of unregister...)
|
|
]
|
|
|
|
|
|
|
|
def init():
|
|
global PPSSPP_EXE
|
|
if not os.path.exists("pspautotests"):
|
|
print("Please run git submodule init; git submodule update;")
|
|
sys.exit(1)
|
|
|
|
if not os.path.exists(TEST_ROOT + "cpu/cpu_alu/cpu_alu.prx"):
|
|
print("Please install the pspsdk and run make in common/ and in all the tests")
|
|
print("(checked for existence of cpu/cpu_alu/cpu_alu.prx)")
|
|
sys.exit(1)
|
|
|
|
for p in PPSSPP_EXECUTABLES:
|
|
if os.path.exists(p):
|
|
PPSSPP_EXE = p
|
|
break
|
|
|
|
if not PPSSPP_EXE:
|
|
print("PPSSPP executable missing, please build one.")
|
|
sys.exit(1)
|
|
|
|
def tcprint(arg):
|
|
global teamcity_mode
|
|
if teamcity_mode:
|
|
print(arg)
|
|
|
|
def run_tests(test_list, args):
|
|
global PPSSPP_EXE, TIMEOUT
|
|
tests_passed = []
|
|
tests_failed = []
|
|
|
|
for test in test_list:
|
|
# Try prx first
|
|
expected_filename = TEST_ROOT + test + ".expected"
|
|
|
|
elf_filename = TEST_ROOT + test + ".prx"
|
|
print(elf_filename)
|
|
|
|
if not os.path.exists(elf_filename):
|
|
print("WARNING: no prx, trying elf")
|
|
elf_filename = TEST_ROOT + test + ".elf"
|
|
|
|
if not os.path.exists(elf_filename):
|
|
print("ERROR: PRX/ELF file missing, failing test: " + test)
|
|
tests_failed.append(test)
|
|
tcprint("##teamcity[testIgnored name='%s' message='PRX/ELF missing']" % test)
|
|
continue
|
|
|
|
if not os.path.exists(expected_filename):
|
|
print("WARNING: expects file missing, failing test: " + test)
|
|
tests_failed.append(test)
|
|
tcprint("##teamcity[testIgnored name='%s' message='Expects file missing']" % test)
|
|
continue
|
|
|
|
expected_output = open(expected_filename).read().strip()
|
|
|
|
tcprint("##teamcity[testStarted name='%s' captureStandardOutput='true']" % test)
|
|
|
|
cmdline = [PPSSPP_EXE, elf_filename]
|
|
cmdline.extend([i for i in args if i not in ['-v', '-g']])
|
|
if os.path.exists(expected_filename + ".bmp"):
|
|
cmdline.extend(["--screenshot=" + expected_filename + ".bmp", "--graphics"])
|
|
|
|
c = Command(cmdline)
|
|
c.run(TIMEOUT)
|
|
|
|
output = c.output.strip()
|
|
|
|
if c.timeout:
|
|
print(output)
|
|
print("Test exceded limit of %d seconds." % TIMEOUT)
|
|
tests_failed.append(test)
|
|
tcprint("##teamcity[testFailed name='%s' message='Test timeout']" % test)
|
|
tcprint("##teamcity[testFinished name='%s']" % test)
|
|
continue
|
|
|
|
if output.startswith("TESTERROR"):
|
|
print("Failed to run test " + elf_filename + "!")
|
|
tests_failed.append(test)
|
|
tcprint("##teamcity[testFailed name='%s' message='Failed to run test']" % test)
|
|
tcprint("##teamcity[testFinished name='%s']" % test)
|
|
continue
|
|
|
|
different = False
|
|
expected_lines = [x.strip() for x in expected_output.splitlines()]
|
|
output_lines = [x.strip() for x in output.splitlines()]
|
|
|
|
for i in range(0, min(len(output_lines), len(expected_lines))):
|
|
if output_lines[i] != expected_lines[i]:
|
|
print("E%i < %s" % (i + 1, expected_lines[i]))
|
|
print("O%i > %s" % (i + 1, output_lines[i]))
|
|
different = True
|
|
|
|
if len(output_lines) != len(expected_lines):
|
|
for i in range(len(output_lines), len(expected_lines)):
|
|
print("E%i < %s" % (i + 1, expected_lines[i]))
|
|
for i in range(len(expected_lines), len(output_lines)):
|
|
print("O%i > %s" % (i + 1, output_lines[i]))
|
|
print("*** Different number of lines!")
|
|
different = True
|
|
|
|
if not different:
|
|
if '-v' in args:
|
|
print("++++++++++++++ The Equal Output +++++++++++++")
|
|
print("\n".join(output_lines))
|
|
print("+++++++++++++++++++++++++++++++++++++++++++++")
|
|
print(" " + test + " - passed!")
|
|
tests_passed.append(test)
|
|
tcprint("##teamcity[testFinished name='%s']" % test)
|
|
else:
|
|
if '-v' in args:
|
|
print("============== output from failed " + test + " :")
|
|
print(output)
|
|
print("============== expected output:")
|
|
print(expected_output)
|
|
print("===============================")
|
|
tests_failed.append(test)
|
|
tcprint("##teamcity[testFailed name='%s' message='Output different from expected file']" % test)
|
|
tcprint("##teamcity[testFinished name='%s']" % test)
|
|
|
|
print("%i tests passed, %i tests failed." % (
|
|
len(tests_passed), len(tests_failed)))
|
|
|
|
if len(tests_failed):
|
|
print("Failed tests:")
|
|
for t in tests_failed:
|
|
print(" " + t)
|
|
print("Ran " + PPSSPP_EXE)
|
|
|
|
|
|
def main():
|
|
global teamcity_mode
|
|
init()
|
|
tests = []
|
|
args = []
|
|
for arg in sys.argv[1:]:
|
|
if arg == '--teamcity':
|
|
teamcity_mode = True
|
|
elif arg[0] == '-':
|
|
args.append(arg)
|
|
else:
|
|
tests.append(arg)
|
|
|
|
if not tests:
|
|
if '-g' in args:
|
|
tests = tests_good
|
|
else:
|
|
tests = tests_next + tests_good
|
|
|
|
run_tests(tests, args)
|
|
|
|
main()
|