mirror of
https://github.com/BepInEx/mono.git
synced 2026-11-18 17:02:10 -10:00
509 lines
20 KiB
Python
Executable File
509 lines
20 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Mimics GNU timeout, but has special modes which gather test result data and retry failed tests.
|
|
|
|
######################################### How this works ##########################################
|
|
#
|
|
# Because we have several different test harnesses and we don't invoke them directly, communication
|
|
# between this script and the harness is done through the simplest means possible (environment
|
|
# variables to communicate babysitter->harness, files in standard locations harness->babysitter).
|
|
#
|
|
# The script supports three different ways of extracting test data from the invoked test suite:
|
|
#
|
|
# 1. "The babysitter protocol": The babysitter sets five environment variables (see below):
|
|
# "Ran test file": A path to a file where the harness should write a line-delimited list of
|
|
# tests which ran to completion.
|
|
# "Failed test file": A path to a file where the harness should write a line-delimited list
|
|
# of tests that failed.
|
|
# "Current test file": A path to a file where the harness should write the currently running
|
|
# test before a test begins, then delete afterward (used to detect early termination).
|
|
# "Run test": A list of test names, used by:
|
|
# "Run mode": This is either RUN or EXCLUDE. If RUN, the test list is a whitelist; run only
|
|
# those tests. If EXCLUDE, the list is a blacklist; run all except those tests.
|
|
# This is the most featureful mode: It can report where we failed in the case of timeouts or
|
|
# crashes that take down the harness, and if the feature is enabled it can retry failed tests.
|
|
# However, it requires modification to the test harness.
|
|
#
|
|
# 2. NUnit XML: The babysitter also sets a sixth environment variable:
|
|
# "XML list file": A path to a file where the harness should write a line-delimited list of
|
|
# paths to NUnit-format XML result files it created.
|
|
# This also requires modification to the test harness, but less of it.
|
|
#
|
|
# 3. NUnit XML (manually specified): If the test harness can't be modified, but the caller of the
|
|
# babysitter script happens to know where the harness writes its result XML files, the caller
|
|
# can specify those paths in the "Extra XML" environment variable (see --help)
|
|
#
|
|
# A single babysitter invocation can currently handle either the babysitter protocol or the XML,
|
|
# but never a mix of the two.
|
|
#
|
|
###################################################################################################
|
|
|
|
import argparse
|
|
import subprocess
|
|
import re
|
|
import signal
|
|
import time
|
|
import sys
|
|
import os.path
|
|
import copy
|
|
import tempfile
|
|
import calendar
|
|
import json
|
|
import platform
|
|
from xml.dom.minidom import parse as xmlparse
|
|
|
|
### Constants
|
|
|
|
# Filenames
|
|
|
|
CURRENT_TEST_FILE = "babysitter_report_current_test_file.txt"
|
|
RAN_TEST_FILE = "babysitter_report_ran_test_file.txt"
|
|
FAILED_TEST_FILE = "babysitter_report_failed_test_file.txt"
|
|
XML_LIST_FILE = "babysitter_report_xml_list_file.txt"
|
|
LOGGING_FILE = "babysitter_report.json_lines"
|
|
|
|
# Environment keys
|
|
|
|
# Keys used for Babysitter<->Nunit IPC
|
|
CURRENT_TEST_KEY = 'MONO_BABYSITTER_NUNIT_CURRENT_TEST_FILE' # Tell nunit where to leave files
|
|
RAN_TEST_KEY = 'MONO_BABYSITTER_NUNIT_RAN_TEST_FILE'
|
|
FAILED_TEST_KEY = 'MONO_BABYSITTER_NUNIT_FAILED_TEST_FILE'
|
|
XML_LIST_KEY = 'MONO_BABYSITTER_NUNIT_XML_LIST_FILE'
|
|
RUN_KEY = 'MONO_BABYSITTER_NUNIT_RUN_TEST' # Semicolon-separated list of test names
|
|
RUN_MODE_KEY = 'MONO_BABYSITTER_NUNIT_RUN_MODE' # Equal to either RUN or EXCLUDE
|
|
|
|
# Keys used for script configuration (see --help text)
|
|
LOG_FILE_KEY = 'MONO_BABYSITTER_LOG_FILE' # Path
|
|
EXTRA_XML_KEY = 'MONO_BABYSITTER_EXTRA_XML' # Semicolon-separated list of paths
|
|
RETRY_KEY = 'MONO_BABYSITTER_RETRY' # Equal to an integer
|
|
VERBOSE_KEY = 'MONO_BABYSITTER_VERBOSE' # "Undocumented"-- used for debugging babysitter
|
|
|
|
# JSON keys
|
|
|
|
DATE_JSON = 'date' # POSIX timestamp of test suite run
|
|
INVOKE_JSON = 'invocation'
|
|
COUNT_JSON = 'iteration' # How many times was command executed?
|
|
LIMIT_JSON = 'failure_max'
|
|
SUPPORT_JSON = 'babysitter_protocol' # Was the test suite running with a babysitter-aware nunit?
|
|
LOADED_XML_JSON = 'loaded_xml' # True if we loaded result XML from the test suite
|
|
FINAL_CODE_JSON = 'final_code'
|
|
TESTS_JSON = 'tests' # Holds dictionary of (test case name)->(dict with TEST_ keys below)
|
|
TEST_FAILURES = 'normal_failures'
|
|
TEST_CRASH_FAILURES = 'crash_failures'
|
|
TEST_TIMEOUT_FAILURES = 'timeout_failures'
|
|
|
|
### Interpret arguments
|
|
|
|
scriptname = sys.argv[0]
|
|
|
|
# This is very silly: This program needs to switch from argparse to manual parsing
|
|
# after the second positional argument, but argparse doesn't let you do this.
|
|
# I work around this with a custom subclass which can selectively swallow errors:
|
|
class Hesitate(Exception):
|
|
pass
|
|
class HesitantParser(argparse.ArgumentParser):
|
|
def __init__(s, *args, **kwargs):
|
|
s.hesitating = True # Errors will be swallowed until this is set false
|
|
argparse.ArgumentParser.__init__(s, *args, **kwargs)
|
|
def error(s, *args, **kwargs):
|
|
if s.hesitating:
|
|
raise Hesitate() # Bail out before errors are printed.
|
|
argparse.ArgumentParser.error(s, *args, **kwargs)
|
|
|
|
# Define args
|
|
argparser = HesitantParser(description="""\
|
|
Run a test suite with a timeout.\n
|
|
Durations are floating point numbers followed by an optional unit:\n
|
|
's' for seconds (the default)
|
|
'm' for minutes
|
|
'h' for hours
|
|
'd' for days\n
|
|
supported environment variables:
|
|
%s: File to write logs to (as line-delimited JSON)
|
|
%s: If set to a number, failed test cases will be rerun this many times (NUnit test suites only)
|
|
%s: Semicolon-separated list of additional NUnit XMLs to check for errors""" %
|
|
(LOG_FILE_KEY, RETRY_KEY, EXTRA_XML_KEY),
|
|
formatter_class=argparse.RawTextHelpFormatter)
|
|
argparser.add_argument('-s', '--signal', dest='signal', metavar='signal', default='TERM',
|
|
help="Send this signal to the command on timeout, instead of TERM.")
|
|
argparser.add_argument('-k', '--kill-after-duration', dest='kill_after', metavar='duration',
|
|
help="If process continues running after signal, send KILL after this duration.")
|
|
argparser.add_argument('duration',
|
|
help="Time to run before sending signal.")
|
|
argparser.add_argument('command', nargs="+", help="Command+args to run.")
|
|
|
|
# Repeatedly parse the given args until we find the shortest prefix for which parsing succeeds.
|
|
argc = len(sys.argv)
|
|
extra_args = []
|
|
for limit in range(1,argc+1):
|
|
try:
|
|
if limit == argc: # On the final pass, parse for real
|
|
argparser.hesitating = False
|
|
args = argparser.parse_args(sys.argv[1:limit])
|
|
# If we're still here, parse_args succeeded.
|
|
# The final parsed arg is the command; remaining argv items are its args.
|
|
extra_args = sys.argv[limit:]
|
|
break
|
|
except Hesitate: # Before the final pass, if parse_args fails, skip
|
|
pass
|
|
|
|
argparser.hesitating = False # Just act like a normal argparser from here
|
|
|
|
durationre = re.compile(r'(\d+)([smhd]?)')
|
|
def parse_duration(duration): # Accept units
|
|
match = durationre.match(duration)
|
|
if not match:
|
|
argparser.error("Could not understand duration %s" % duration)
|
|
time, units = match.group(1), match.group(2)
|
|
time = int(time)
|
|
if units == 'm':
|
|
time *= 60
|
|
elif units == 'h':
|
|
time *= 60*60
|
|
elif units == 'd':
|
|
time *= 60*60*24
|
|
return time
|
|
|
|
def parse_signal(sig): # Accept names
|
|
if sig.isdigit():
|
|
return int(sig)
|
|
for k,v in signal.__dict__.items():
|
|
if k == ("SIG%s" % sig):
|
|
return v
|
|
argparser.error("Could not understand signal name %s" % sig)
|
|
|
|
# Final interpretation of arguments
|
|
duration = parse_duration(args.duration)
|
|
kill_after = parse_duration(args.kill_after) if args.kill_after is not None else None
|
|
timeout_signal = parse_signal(args.signal)
|
|
command = args.command + extra_args
|
|
|
|
# If we are running in Cygwin, Python will believe it is a UNIX application but Mono will be Windows.
|
|
|
|
cygwin = platform.system().startswith("CYGWIN")
|
|
|
|
def outgoingPath(path):
|
|
if cygwin: # Invoke cygpath and strip newline
|
|
return subprocess.check_output(["cygpath", "-w", path])[:-1]
|
|
return path
|
|
|
|
def incomingPath(path):
|
|
if cygwin:
|
|
return subprocess.check_output(["cygpath", path])[:-1]
|
|
return path
|
|
|
|
# Some of the things we put in global_env are paths. If we're in cygwin, we have to keep separate
|
|
# local-use and env (mono use) copies of these keys.
|
|
env_source = {}
|
|
|
|
# Process environment
|
|
global_env = copy.deepcopy( os.environ )
|
|
|
|
verbose = VERBOSE_KEY in global_env
|
|
logging = LOG_FILE_KEY in global_env
|
|
logfile = global_env[LOG_FILE_KEY] if logging else None
|
|
xml_list = global_env[EXTRA_XML_KEY].split(";") if EXTRA_XML_KEY in global_env and global_env[EXTRA_XML_KEY] else []
|
|
crash_resuming = True # TODO: Consider exposing this option, or adding a retry_on_crash option.
|
|
failmax = int(global_env[RETRY_KEY]) if RETRY_KEY in global_env else 0
|
|
babysitting = True # If false, babysitter becomes a timeout clone with no env manipulation or anything.
|
|
if babysitting:
|
|
babysitter_dir = tempfile.mkdtemp()
|
|
env_source[CURRENT_TEST_KEY] = os.path.join(babysitter_dir, CURRENT_TEST_FILE)
|
|
env_source[RAN_TEST_KEY] = os.path.join(babysitter_dir, RAN_TEST_FILE)
|
|
env_source[FAILED_TEST_KEY] = os.path.join(babysitter_dir, FAILED_TEST_FILE)
|
|
env_source[XML_LIST_KEY] = os.path.join(babysitter_dir, XML_LIST_FILE)
|
|
|
|
env_source_keys = [CURRENT_TEST_KEY, RAN_TEST_KEY, FAILED_TEST_KEY, XML_LIST_KEY]
|
|
for key in env_source_keys:
|
|
global_env[key] = outgoingPath(env_source[key])
|
|
|
|
have_unix_process_groups = 'killpg' in os.__dict__
|
|
have_windows_process_groups = 'CREATE_NEW_PROCESS_GROUP' in subprocess.__dict__
|
|
|
|
### Timeout implementation
|
|
|
|
def wait(proc, duration):
|
|
# TODO: If we detect Python 3.3, Popen objects have a wait(timeout) method we can use
|
|
start = time.time()
|
|
while True:
|
|
code = proc.poll()
|
|
if code is not None:
|
|
return code
|
|
if time.time()-start > duration:
|
|
return None
|
|
time.sleep(0.05)
|
|
|
|
# Popen and send_signal can't be called in their basic forms because we want to
|
|
# send signals to all children, not just to the immediately spawned process.
|
|
# Unfortunately the way to do this varies by operating system.
|
|
def popen(*args, **kwargs):
|
|
if have_unix_process_groups: # Call function on spawn to become process group leader
|
|
kwargs['preexec_fn'] = os.setsid
|
|
elif have_windows_process_groups: # Set magic flag for Windows process groups
|
|
kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
|
|
return subprocess.Popen(*args, **kwargs)
|
|
|
|
def send_signal(proc, sig):
|
|
if have_unix_process_groups: # UNIX
|
|
# For compatibility with GNU timeout, pre-send the signal to just the monitored process
|
|
os.kill(proc.pid, sig)
|
|
# Send signal to entire group
|
|
try:
|
|
os.killpg(proc.pid, sig)
|
|
except OSError as e:
|
|
sys.stderr.write("%s: Warning, could not kill process group %s because %s\n" % (scriptname, proc.pid, e))
|
|
# For compatibility with GNU Timeout, send a SIGCONT after the signal
|
|
# (so delivery has a chance to occur even for stopped processes)
|
|
if sig != signal.SIGKILL and sig != signal.SIGCONT:
|
|
os.kill(proc.pid, signal.SIGCONT)
|
|
elif have_windows_process_groups: # Windows with Python 2.7 or better
|
|
os.kill(proc.pid, sig) # Becuase CREATE_NEW_PROCESS_GROUP, will go to entire group
|
|
else: # Windows with Python 2.6-- CREATE_NEW_PROCESS_GROUP not supported
|
|
proc.send_signal(sig) # No way to contact group, just kill process
|
|
|
|
### Utility functions
|
|
|
|
def attemptDelete(path):
|
|
try:
|
|
os.remove(path)
|
|
except OSError:
|
|
pass
|
|
|
|
def attemptLines(path):
|
|
try:
|
|
with open(path) as f:
|
|
return map(lambda s: s.strip('\r\n'), f.readlines())
|
|
except (OSError, IOError):
|
|
return []
|
|
|
|
def attemptFirstLine(path):
|
|
lines = attemptLines(path)
|
|
if lines:
|
|
return lines[0]
|
|
return None
|
|
|
|
def posixtime(): # Amazingly, time.time() does not guarantee an epoch in the docs. However this does:
|
|
return calendar.timegm(time.gmtime())
|
|
|
|
failcount = {}
|
|
def failure_may_retry(test):
|
|
if test not in failcount:
|
|
failcount[test] = 0
|
|
failcount[test] += 1
|
|
return failcount[test] < failmax
|
|
|
|
def verbose_print(arg):
|
|
if (verbose):
|
|
print(arg)
|
|
|
|
def failure_annotate(test):
|
|
return "%s (failure #%d of %d allowed)" % (test, failcount[test], failmax)
|
|
|
|
def pluralize(lst):
|
|
return "s" if len(lst) > 1 else ""
|
|
|
|
### Run command
|
|
|
|
def run(): # Returns exit code
|
|
resume_after = []
|
|
retry_these = []
|
|
ever_completed = False
|
|
died_politely = False
|
|
proc = None
|
|
code = None
|
|
|
|
# Set up logging
|
|
log = {DATE_JSON: posixtime(), COUNT_JSON:0, LIMIT_JSON:failmax, SUPPORT_JSON:False,
|
|
LOADED_XML_JSON:False, INVOKE_JSON: " ".join(command)}
|
|
|
|
def log_value(key, set=None, add=None, target=log): # Call to add toplevel value to log
|
|
if add is not None:
|
|
if key not in target:
|
|
target[key] = 0
|
|
target[key] += add
|
|
else:
|
|
target[key] = set
|
|
|
|
def log_test(testname, key, set=None, add=None): # Call to add test-case-level value to log
|
|
if TESTS_JSON not in log:
|
|
log[TESTS_JSON] = {}
|
|
if testname not in log[TESTS_JSON]:
|
|
log[TESTS_JSON][testname] = {}
|
|
log_value(key, set=set, add=add, target=log[TESTS_JSON][testname])
|
|
|
|
# Ready to run tests
|
|
try:
|
|
while True:
|
|
env = copy.copy(global_env)
|
|
if ever_completed:
|
|
retry_next = []
|
|
else: # Persist reported failures list until first non-crash run
|
|
retry_next = retry_these
|
|
|
|
log_value(COUNT_JSON, add=1)
|
|
|
|
# Prepare environment/filesystem
|
|
if babysitting:
|
|
for key in env_source_keys: # Clear all paths intended for use by mono
|
|
attemptDelete(env_source[key])
|
|
if resume_after:
|
|
env[RUN_KEY] = ";".join(resume_after)
|
|
env[RUN_MODE_KEY] = "EXCLUDE"
|
|
elif retry_these:
|
|
env[RUN_KEY] = ";".join(retry_these)
|
|
env[RUN_MODE_KEY] = "RUN"
|
|
|
|
# Run test suite
|
|
try:
|
|
proc = popen(command, env=env)
|
|
except OSError:
|
|
died_politely = True
|
|
sys.stderr.write("%s: Could not execute command `%s`\n" % (scriptname, command[0]))
|
|
sys.exit(127)
|
|
|
|
code = wait(proc, duration)
|
|
timed_out = code is None
|
|
if timed_out: # Initial timeout
|
|
sys.stderr.write("%s: Sending %s to `%s`\n" % (scriptname, "SIG" + args.signal, command[0]))
|
|
send_signal(proc, timeout_signal)
|
|
if kill_after is not None: # Kill-after timeout
|
|
code = wait(proc, kill_after)
|
|
if code is None:
|
|
sys.stderr.write("%s: Sending SIGKILL to `%s`\n" % (scriptname, command[0]))
|
|
send_signal(proc, signal.SIGKILL)
|
|
code = wait(proc, 1)
|
|
sys.stderr.write("%s: Command `%s` returned with %s after SIGKILL\n" % (scriptname, command[0], str(code)))
|
|
else:
|
|
sys.stderr.write("%s: Command `%s` returned with %s after %s\n" % (scriptname, command[0], str(code), "SIG" + args.signal))
|
|
code = proc.wait() # Waits forever
|
|
sys.stderr.write("%s: Command `%s` timed out with %s\n" % (scriptname, command[0], str(code)))
|
|
died_politely = True
|
|
|
|
# The test suite has now run, and what happens next varies:
|
|
# 1. The suite either completed fully without failures, or timed out: Just quit.
|
|
# 2. The suite crashed (halted without completing):
|
|
# Remember any failures for later and rerun, using a blacklist of testcases we have completed.
|
|
# 3. The suite completed, but there were failures reported:
|
|
# Rerun, using a whitelist of only reported-failed testcases.
|
|
# 4. The suite crashed partway through a run with a whitelist:
|
|
# Rerun, using a whitelist consisting of the previous whitelist minus successful testcases.
|
|
|
|
crashed_at = attemptFirstLine(env_source[CURRENT_TEST_KEY])
|
|
failed_tests = attemptLines(env_source[FAILED_TEST_KEY])
|
|
ran_tests = attemptLines(env_source[RAN_TEST_KEY])
|
|
wrote_xml = attemptLines(env_source[XML_LIST_KEY])
|
|
bailout = False
|
|
|
|
if crashed_at or failed_tests or ran_tests: # Test suite follows the babysitter protocol
|
|
log_value(SUPPORT_JSON, True)
|
|
|
|
if not crashed_at and not ever_completed: # The resume_after whitelist is only
|
|
resume_after = [] # used before the first noncrashing run
|
|
ever_completed = True
|
|
|
|
if timed_out: # Currently no retries after timeout
|
|
bailout = True
|
|
code = 124 # See GNU timeout manpage
|
|
|
|
if code or crashed_at: # Process failures
|
|
# Handle crash failures
|
|
if crashed_at and not timed_out:
|
|
log_test(crashed_at, TEST_CRASH_FAILURES, add=1)
|
|
if not crash_resuming:
|
|
bailout = True
|
|
|
|
if failure_may_retry(crashed_at):
|
|
if ever_completed: # Rerun with whitelist next time
|
|
for test in retry_these: # Prepopulate with last whitelist minus run testcases
|
|
if test == crashed_at or test not in ran_tests: # (plus crashed testcase)
|
|
retry_next.append(test)
|
|
else: # Rerun with blacklist next time
|
|
for test in ran_tests: # Add testcases we just ran to blacklist
|
|
if test != crashed_at: # (except for the crashed testcase)
|
|
resume_after.append(test)
|
|
else:
|
|
bailout = True
|
|
|
|
# Handle reported failures
|
|
for test in failed_tests:
|
|
log_test(test, TEST_FAILURES, add=1)
|
|
if failure_may_retry(test):
|
|
retry_next.append(test)
|
|
else:
|
|
bailout = True
|
|
|
|
# Report human-readable failures for stdout log.
|
|
message = "%s:" % (scriptname)
|
|
if timed_out:
|
|
message += " Saw timeout in test case %s (never allowed)." % (crashed_at)
|
|
elif crashed_at:
|
|
message += " Saw crash in test case %s." % (failure_annotate(crashed_at))
|
|
if failed_tests:
|
|
message += " Saw test failure in test case%s %s." % (pluralize(failed_tests), "; ".join(map(failure_annotate, failed_tests)))
|
|
if not (timed_out or crashed_at or failed_tests):
|
|
message += " Test suite terminated with code %d, " % (code)
|
|
if log[SUPPORT_JSON]:
|
|
message += "but failure did not occur during a test case. Halting."
|
|
elif xml_list or wrote_xml:
|
|
message += "will extract test results from XML. Halting."
|
|
else:
|
|
message += "and suite cannot report test case data. Halting."
|
|
elif bailout:
|
|
message += " Will halt testing."
|
|
print(message)
|
|
|
|
if not log[SUPPORT_JSON]:
|
|
for xml in (xml_list + [incomingPath(xml) for xml in wrote_xml]):
|
|
verbose_print("Will attempt to load XML from %s" % (xml))
|
|
try:
|
|
data = xmlparse(xml).documentElement
|
|
if data.nodeName != 'test-results':
|
|
raise ValueError("Toplevel element was not <test-results />")
|
|
|
|
log_value(LOADED_XML_JSON, True)
|
|
|
|
search = [data]
|
|
while search:
|
|
nextsearch = []
|
|
for node in search:
|
|
for child in node.childNodes:
|
|
if child.nodeName == 'test-suite' or child.nodeName == 'results':
|
|
nextsearch.append(child) # Descend
|
|
elif child.nodeName == 'test-case':
|
|
name = child.getAttribute("name")
|
|
if child.getAttribute("executed") == "True" and child.getAttribute("success") != "True":
|
|
log_test(name, TEST_FAILURES, add=1)
|
|
|
|
search = nextsearch
|
|
except Exception as e:
|
|
print("Could not load XML file %s. Reason: %s" % (xml, e))
|
|
data
|
|
|
|
if bailout or not (resume_after or retry_next): # If not retrying
|
|
return code
|
|
|
|
# If we got here, a retry is occurring
|
|
retry_these = retry_next
|
|
|
|
# Report human-readable retry notice for stdout log.
|
|
message = "%s: Will rerun test suite" % (scriptname)
|
|
if log[COUNT_JSON] > 1:
|
|
message += " (iteration #%d)" % (log[COUNT_JSON])
|
|
if resume_after:
|
|
message += ", resuming at crashed testcase %s." % (crashed_at)
|
|
else:
|
|
message += ", running previously failed testcase%s: %s." % (pluralize(retry_these), "; ".join(retry_these))
|
|
print(message)
|
|
finally:
|
|
# Emergency: Ensure command does not outlive this script
|
|
if proc is not None and not died_politely:
|
|
send_signal(proc, signal.SIGKILL)
|
|
|
|
# Write out logs
|
|
log_value(FINAL_CODE_JSON, "EXCEPTION" if code is None else code)
|
|
if logging:
|
|
with open(logfile, "a") as f:
|
|
f.write(json.dumps(log) + os.linesep)
|
|
|
|
sys.exit( run() )
|