Files
RetroArch/tools/ci_local.py
LibretroAdmin 58106ef61c CI: skip the workflows a push's samples and workflow files do not touch
Every push ran every workflow. tools/ci_paths.py now writes each
workflow's push and pull_request paths filter: everything, less the
samples trees and the other workflow files, plus the sample
directories its jobs build or run (through the scripts they call and
the paths those directories name in turn) and the workflow itself. A
push to one sample runs the full-tree builds no longer, only the
workflows that run that sample; a push to one workflow runs it alone;
any other change runs everything as before. Jobs that call a tool or
CI script, walk the samples trees or read the workflow files keep
those whole, and workflows with a filter of their own keep it.

The CI local runner workflow runs tools/ci_paths.py --check, which
fails when a filter no longer matches what its workflow names, and
tools/ci_local.py which reads past a generated filter to the coverage.
2026-11-04 16:30:17 +01:00

2002 lines
68 KiB
Python
Executable File

#!/usr/bin/env python3
"""Run the GitHub CI workflows locally, and find the ones that cover a file.
A workflow's steps are its run: blocks, run here in order with the same
shell, environment, working directories and ${{ }} expressions GitHub
gives them, so a local result is the CI result rather than an
approximation of it. Nothing outside the Python standard library is
needed; the YAML the workflows use is read by the small parser below.
tools/ci_local.py list
Every workflow and job, with its runner and matrix size.
tools/ci_local.py which <path>...
The jobs that build, run or check the given files: those that name
them (directly, through a script they call, or through a sample's
Makefile), the full-tree builds, and the tree-wide checkers.
tools/ci_local.py run <workflow> [--job ID] [--matrix KEY=VALUE]...
[--skip REGEX]... [--dry-run] [--clean-copy]
[--host] [--force] [--times]
Runs a workflow's jobs, or one job; every matrix combination unless
--matrix narrows them. <workflow> is a file name, with or without
.yml, or the workflow's name.
--dry-run print each step's resolved script without running it
--skip REGEX skip steps whose name matches (e.g. "^Install")
--clean-copy run in a fresh worktree of HEAD plus the uncommitted
changes, leaving this checkout untouched
--host run a container job on this machine instead of in
its image (docker or podman is used otherwise)
--force run a job whose runner is another OS, or that uses
an action this script cannot stand in for
--times print every step's time, not only the three slowest
tools/ci_local.py selftest
Checks the parser against PyYAML (when installed) and resolves every
job of every workflow in a dry run.
Actions are not run. actions/checkout is the checkout itself (another
repository is cloned instead), setup-python is the host's Python, and
setup-emsdk verifies that the host emcc matches the pinned version.
Cache and artifact steps are skipped. A job using any other action is
refused unless --force is given, which skips it.
"""
import fnmatch, json, os, re, shlex, shutil, subprocess, sys, tempfile, time
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
WF_DIR = os.path.join(ROOT, '.github', 'workflows')
# --------------------------------------------------------------------------
# YAML: the block subset the workflows use, resolved as PyYAML resolves it
# --------------------------------------------------------------------------
class YamlError(Exception):
pass
_NULL = re.compile(r'^(?:~|null|Null|NULL|)$')
_BOOL = {'yes': True, 'Yes': True, 'YES': True, 'no': False, 'No': False,
'NO': False, 'true': True, 'True': True, 'TRUE': True,
'false': False, 'False': False, 'FALSE': False, 'on': True,
'On': True, 'ON': True, 'off': False, 'Off': False, 'OFF': False}
_INT = re.compile(r'^(?:[-+]?0b[0-1_]+|[-+]?0[0-7_]+|[-+]?(?:0|[1-9][0-9_]*)'
r'|[-+]?0x[0-9a-fA-F_]+|[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$')
_FLOAT = re.compile(r'^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)?'
r'|\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?'
r'|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*'
r'|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$')
def _resolve(s):
if _NULL.match(s):
return None
if s in _BOOL:
return _BOOL[s]
if _INT.match(s):
t = s.replace('_', '')
sign = -1 if t[0] == '-' else 1
if t[0] in '+-':
t = t[1:]
if t == '0':
return 0
if t.startswith('0b'):
return sign * int(t[2:], 2)
if t.startswith('0x'):
return sign * int(t[2:], 16)
if ':' in t:
v = 0
for part in t.split(':'):
v = v * 60 + int(part)
return sign * v
if t[0] == '0':
return sign * int(t, 8)
return sign * int(t)
if _FLOAT.match(s):
t = s.replace('_', '').lower()
if t.endswith('.inf'):
return float('-inf') if t[0] == '-' else float('inf')
if t.endswith('.nan'):
return float('nan')
if ':' in t:
sign = -1 if t[0] == '-' else 1
v = 0.0
for part in t.lstrip('+-').split(':'):
v = v * 60 + float(part)
return sign * v
return float(t)
return s
def _strip_comment(s):
"""Cut a trailing ' #' comment, outside quotes."""
q = None
i = 0
while i < len(s):
c = s[i]
if q:
if c == '\\' and q == '"':
i += 1
elif c == q:
q = None
elif c in '"\'' and (i == 0 or s[i - 1] in ' \t[{,:'):
q = c
elif c == '#' and (i == 0 or s[i - 1] in ' \t'):
return s[:i].rstrip()
i += 1
return s.rstrip()
def _indent(line):
return len(line) - len(line.lstrip(' '))
_ESC = {'0': '\0', 'a': '\a', 'b': '\b', 't': '\t', '\t': '\t', 'n': '\n',
'v': '\v', 'f': '\f', 'r': '\r', 'e': '\x1b', ' ': ' ', '"': '"',
'/': '/', '\\': '\\', 'N': '\x85', '_': '\xa0', 'L': '\u2028',
'P': '\u2029'}
def _fold(parts):
"""Join the lines of a multi-line flow scalar."""
out = parts[0]
pending = 0
for p in parts[1:]:
if p == '':
pending += 1
continue
out += ('\n' * pending) if pending else ' '
pending = 0
out += p
if pending:
out += '\n' * pending
return out
class _Parser:
def __init__(self, text):
self.lines = text.split('\n')
def blank(self, i):
s = self.lines[i].strip()
return s == '' or s.startswith('#') or s == '---'
def next_content(self, i):
while i < len(self.lines) and self.blank(i):
i += 1
return i
def parse(self):
i = self.next_content(0)
if i >= len(self.lines):
return None
node, i = self.node(i, -1)
if self.next_content(i) < len(self.lines):
raise YamlError('unparsed content at line %d' % (i + 1))
return node
def node(self, i, parent):
i = self.next_content(i)
if i >= len(self.lines):
return None, i
ind = _indent(self.lines[i])
if ind <= parent:
return None, i
body = self.lines[i][ind:]
if body == '-' or body.startswith('- '):
return self.seq(i, ind)
if self.key(body) is not None:
return self.mapping(i, ind)
return self.scalar(i, ind, body, parent)
def key(self, body):
"""(key, rest) when the line is a mapping entry, else None."""
if body[:1] in '"\'':
q = body[0]
j = 1
while j < len(body):
if q == '"' and body[j] == '\\':
j += 2
continue
if body[j] == q:
if q == "'" and body[j + 1:j + 2] == "'":
j += 2
continue
break
j += 1
rest = body[j + 1:].lstrip(' ')
if not rest.startswith(':') or rest[1:2] not in ('', ' '):
return None
return self.flow_scalar(body[:j + 1]), rest[1:].strip()
if body[:1] in '[{#|>!&*%@`':
return None
m = re.match(r'^([^#]*?)\s*:(?:\s|$)', body)
if not m:
return None
return _resolve(m.group(1)), body[m.end():].strip()
def mapping(self, i, ind):
out = {}
while True:
i = self.next_content(i)
if i >= len(self.lines) or _indent(self.lines[i]) != ind:
break
body = self.lines[i][ind:]
if body == '-' or body.startswith('- '):
break
kv = self.key(body)
if kv is None:
raise YamlError('expected a key at line %d' % (i + 1))
k, rest = kv
rest = _strip_comment(rest)
if rest == '':
j = self.next_content(i + 1)
if j < len(self.lines) and _indent(self.lines[j]) == ind \
and (self.lines[j][ind:] == '-'
or self.lines[j][ind:].startswith('- ')):
v, i = self.seq(j, ind)
else:
v, i = self.node(i + 1, ind)
else:
v, i = self.value(i, ind, rest)
out[k] = v
return out, i
def seq(self, i, ind):
out = []
while True:
i = self.next_content(i)
if i >= len(self.lines) or _indent(self.lines[i]) != ind:
break
body = self.lines[i][ind:]
if not (body == '-' or body.startswith('- ')):
break
rest = body[1:]
if rest.strip() == '' or rest.strip().startswith('#'):
v, i = self.node(i + 1, ind)
else:
col = ind + 1 + _indent(rest)
self.lines[i] = ' ' * col + rest.lstrip(' ')
v, i = self.node(i, ind)
out.append(v)
return out, i
def scalar(self, i, ind, body, parent):
return self.value(i, parent, _strip_comment(body))
def value(self, i, ind, rest):
"""A value that starts on line i after a key (or a bare item)."""
if rest[:1] in '|>':
return self.block_scalar(i, ind, rest)
if rest[:1] in '[{':
text = rest
j = i
while not self.balanced(text):
j += 1
if j >= len(self.lines):
raise YamlError('unclosed flow collection at line %d' % (i + 1))
text += ' ' + _strip_comment(self.lines[j].strip())
v, pos = self.flow(text, 0)
return v, j + 1
if rest[:1] in '"\'':
text = rest
j = i
while not self.quote_closed(text):
j += 1
if j >= len(self.lines):
raise YamlError('unclosed quote at line %d' % (i + 1))
text += '\n' + self.lines[j]
return self.flow_scalar(text), j + 1
parts = [rest]
j = i + 1
while j < len(self.lines):
line = self.lines[j]
s = line.strip()
if s == '':
parts.append('')
j += 1
continue
if _indent(line) <= ind or s.startswith('#'):
break
parts.append(_strip_comment(s))
j += 1
while len(parts) > 1 and parts[-1] == '':
parts.pop()
j -= 1
while j > i + 1 and self.lines[j - 1].strip() == '':
j -= 1
return _resolve(_fold(parts)), j
def quote_closed(self, text):
q = text[0]
j = 1
while j < len(text):
if q == '"' and text[j] == '\\':
j += 2
continue
if text[j] == q:
if q == "'" and text[j + 1:j + 2] == "'":
j += 2
continue
return True
j += 1
return False
def balanced(self, text):
depth = 0
q = None
for k, c in enumerate(text):
if q:
if c == q:
q = None
elif c in '"\'' and (k == 0 or text[k - 1] in ' [{,:'):
q = c
elif c in '[{':
depth += 1
elif c in ']}':
depth -= 1
if depth == 0:
return True
return False
def block_scalar(self, i, ind, header):
m = re.match(r'^([|>])([+-]?)([1-9]?)([+-]?)\s*(?:#.*)?$', header)
if not m:
raise YamlError('bad block scalar header at line %d' % (i + 1))
style, chomp, digit = m.group(1), m.group(2) or m.group(4), m.group(3)
j = i + 1
if digit:
content = max(ind, 0) + int(digit)
else:
k = j
while k < len(self.lines) and self.lines[k].strip() == '':
k += 1
content = _indent(self.lines[k]) if k < len(self.lines) else 0
if content <= ind:
content = ind + 1
body = []
while j < len(self.lines):
line = self.lines[j]
if line.strip() == '':
body.append(line[content:] if len(line) > content else '')
j += 1
continue
if _indent(line) < content:
break
body.append(line[content:])
j += 1
trailing = 0
while body and body[-1].strip() == '':
body.pop()
trailing += 1
end = j - trailing
if style == '|':
text = '\n'.join(body)
else:
text = ''
prev_more = True
for n, line in enumerate(body):
more = line[:1] in (' ', '\t')
if n == 0:
text = line
elif line == '':
text += '\n'
elif text.endswith('\n') or more or prev_more:
text += ('\n' if not text.endswith('\n') else '') + line
else:
text += ' ' + line
prev_more = more or line == ''
if chomp == '-':
out = text
elif chomp == '+':
out = (text + '\n' if body else '') + '\n' * trailing
else:
out = text + '\n' if body else ''
return out, end
def flow_scalar(self, text):
text = text.strip()
q = text[0]
if q == "'":
end = len(text) - 1
parts = text[1:end].split('\n')
if len(parts) > 1:
parts = [parts[0].rstrip(' ')] + [p.strip(' ') for p in parts[1:-1]] \
+ [parts[-1].lstrip(' ')]
return _fold(parts).replace("''", "'")
out = ''
j = 1
lines = []
while j < len(text):
c = text[j]
if c == '\\':
n = text[j + 1]
if n == '\n':
j += 2
while j < len(text) and text[j] in ' \t':
j += 1
out += '\x00JOIN'
continue
if n == 'x':
out += chr(int(text[j + 2:j + 4], 16))
j += 4
continue
if n == 'u':
out += chr(int(text[j + 2:j + 6], 16))
j += 6
continue
if n == 'U':
out += chr(int(text[j + 2:j + 10], 16))
j += 10
continue
out += _ESC[n]
j += 2
continue
if c == '"':
break
out += c
j += 1
parts = out.split('\n')
if len(parts) > 1:
parts = [parts[0].rstrip(' ')] + [p.strip(' ') for p in parts[1:-1]] \
+ [parts[-1].lstrip(' ')]
out = _fold(parts)
return out.replace('\x00JOIN', '')
def flow(self, s, p):
p = self.ws(s, p)
c = s[p]
if c == '[':
out = []
p += 1
while True:
p = self.ws(s, p)
if s[p] == ']':
return out, p + 1
v, p = self.flow(s, p)
out.append(v)
p = self.ws(s, p)
if s[p] == ',':
p += 1
if c == '{':
out = {}
p += 1
while True:
p = self.ws(s, p)
if s[p] == '}':
return out, p + 1
k, p = self.flow_atom(s, p, key=True)
p = self.ws(s, p)
v = None
if s[p] == ':':
p = self.ws(s, p + 1)
if s[p] not in ',}':
v, p = self.flow(s, p)
out[k] = v
p = self.ws(s, p)
if s[p] == ',':
p += 1
return self.flow_atom(s, p)
def ws(self, s, p):
while p < len(s) and s[p] in ' \t\n':
p += 1
return p
def flow_atom(self, s, p, key=False):
if s[p] in '"\'':
q = s[p]
j = p + 1
while j < len(s):
if q == '"' and s[j] == '\\':
j += 2
continue
if s[j] == q:
if q == "'" and s[j + 1:j + 2] == "'":
j += 2
continue
break
j += 1
return self.flow_scalar(s[p:j + 1]), j + 1
j = p
while j < len(s):
if s[j] in ',]}':
break
if s[j] == ':' and (j + 1 >= len(s) or s[j + 1] in ' ,]}'):
break
j += 1
return _resolve(s[p:j].strip()), j
def yaml_load(text):
return _Parser(text).parse()
# --------------------------------------------------------------------------
# ${{ }} expressions
# --------------------------------------------------------------------------
class ExprError(Exception):
pass
class Unresolved(Exception):
"""A value that only exists while the job runs (a step's output)."""
_TOKEN = re.compile(r"\s*(?:(?P<num>-?(?:0x[0-9a-fA-F]+|[0-9]+(?:\.[0-9]+)?"
r"(?:[eE][-+]?[0-9]+)?))|(?P<str>'(?:[^']|'')*')"
r"|(?P<op>==|!=|<=|>=|&&|\|\||[<>!()\[\],.*])"
r"|(?P<id>[A-Za-z_][A-Za-z0-9_-]*))")
def _tokens(src):
out = []
p = 0
src = src.strip()
while p < len(src):
m = _TOKEN.match(src, p)
if not m or m.end() == p:
raise ExprError('cannot read expression: %s' % src)
for k in ('num', 'str', 'op', 'id'):
if m.group(k) is not None:
out.append((k, m.group(k)))
break
p = m.end()
while p < len(src) and src[p] in ' \t\n':
p += 1
return out
def to_str(v):
if v is None:
return ''
if v is True:
return 'true'
if v is False:
return 'false'
if isinstance(v, float):
if v != v:
return 'NaN'
if v == int(v) and abs(v) < 1e15:
return str(int(v))
return repr(v)
if isinstance(v, (dict, list)):
return json.dumps(v, indent=2)
return str(v)
def to_num(v):
if v is None:
return 0.0
if isinstance(v, bool):
return 1.0 if v else 0.0
if isinstance(v, (int, float)):
return float(v)
if isinstance(v, str):
s = v.strip()
if s == '':
return 0.0
try:
return float(int(s, 16)) if s.lower().startswith('0x') else float(s)
except ValueError:
return float('nan')
return float('nan')
def truthy(v):
if v is None or v is False:
return False
if isinstance(v, (int, float)) and not isinstance(v, bool):
return v == v and v != 0
if isinstance(v, str):
return v != ''
return True
def loose_eq(a, b):
if isinstance(a, str) and isinstance(b, str):
return a.lower() == b.lower()
if isinstance(a, (dict, list)) or isinstance(b, (dict, list)):
return a is b
if type(a) is type(b) and not isinstance(a, str):
return a == b
return to_num(a) == to_num(b)
class Expr:
"""Evaluates one expression against the run's contexts."""
FUNCS = ('success', 'failure', 'always', 'cancelled', 'startswith',
'endswith', 'contains', 'format', 'join', 'tojson', 'fromjson',
'hashfiles')
def __init__(self, ctx, status):
self.ctx = ctx
self.status = status
def eval(self, src):
self.toks = _tokens(src)
self.p = 0
v = self.or_()
if self.p != len(self.toks):
raise ExprError('trailing input in: %s' % src)
return v
def peek(self):
return self.toks[self.p] if self.p < len(self.toks) else (None, None)
def take(self, val=None):
t = self.peek()
if val is not None and t[1] != val:
raise ExprError('expected %s' % val)
self.p += 1
return t
def or_(self):
v = self.and_()
while self.peek()[1] == '||':
self.take()
r = self.and_()
v = v if truthy(v) else r
return v
def and_(self):
v = self.cmp()
while self.peek()[1] == '&&':
self.take()
r = self.cmp()
v = r if truthy(v) else v
return v
def cmp(self):
v = self.unary()
while self.peek()[1] in ('==', '!=', '<', '>', '<=', '>='):
op = self.take()[1]
r = self.unary()
if op == '==':
v = loose_eq(v, r)
elif op == '!=':
v = not loose_eq(v, r)
else:
a, b = to_num(v), to_num(r)
v = {'<': a < b, '>': a > b, '<=': a <= b, '>=': a >= b}[op]
return v
def unary(self):
if self.peek()[1] == '!':
self.take()
return not truthy(self.unary())
return self.postfix()
def postfix(self):
kind, val = self.peek()
if val == '(':
self.take()
v = self.or_()
self.take(')')
elif kind == 'num':
self.take()
v = float(int(val, 16)) if 'x' in val.lower() else float(val)
elif kind == 'str':
self.take()
v = val[1:-1].replace("''", "'")
elif kind == 'id':
self.take()
low = val.lower()
if low in ('true', 'false'):
return low == 'true'
if low == 'null':
return None
if self.peek()[1] == '(':
v = self.call(low)
elif val not in self.ctx:
raise ExprError('unknown context: %s' % val)
else:
v = self.ctx[val]
else:
raise ExprError('unexpected %s' % val)
while self.peek()[1] in ('.', '['):
if self.take()[1] == '.':
k = self.take()
if k[0] != 'id' and k[1] != '*':
raise ExprError('bad property %s' % k[1])
v = self.prop(v, k[1])
else:
k = self.or_()
self.take(']')
v = self.prop(v, k if isinstance(k, str) else int(to_num(k)))
return v
def prop(self, v, k):
if isinstance(v, Unresolved):
return v
if isinstance(v, dict):
if isinstance(k, str):
for key in v:
if isinstance(key, str) and key.lower() == k.lower():
return v[key]
return None
if isinstance(v, list) and isinstance(k, int):
return v[k] if 0 <= k < len(v) else None
return None
def call(self, name):
self.take('(')
args = []
while self.peek()[1] != ')':
args.append(self.or_())
if self.peek()[1] == ',':
self.take()
self.take(')')
if name not in self.FUNCS:
raise ExprError('unknown function: %s' % name)
if name == 'success':
return self.status == 'success'
if name == 'failure':
return self.status == 'failure'
if name == 'always':
return True
if name == 'cancelled':
return False
for a in args:
if isinstance(a, Unresolved):
raise a
if name == 'startswith':
return to_str(args[0]).lower().startswith(to_str(args[1]).lower())
if name == 'endswith':
return to_str(args[0]).lower().endswith(to_str(args[1]).lower())
if name == 'contains':
if isinstance(args[0], list):
return any(loose_eq(x, args[1]) for x in args[0])
return to_str(args[1]).lower() in to_str(args[0]).lower()
if name == 'format':
fmt = to_str(args[0])
return re.sub(r'\{(\d+)\}', lambda m: to_str(args[int(m.group(1)) + 1]),
fmt).replace('{{', '{').replace('}}', '}')
if name == 'join':
sep = to_str(args[1]) if len(args) > 1 else ','
if isinstance(args[0], list):
return sep.join(to_str(x) for x in args[0])
return to_str(args[0])
if name == 'tojson':
return json.dumps(args[0], indent=2)
if name == 'fromjson':
return json.loads(to_str(args[0]))
return ''
_TEMPLATE = re.compile(r'\$\{\{((?:(?!\}\}).)*)\}\}', re.S)
def interpolate(text, ctx, status='success'):
if not isinstance(text, str):
return text
def sub(m):
v = Expr(ctx, status).eval(m.group(1))
if isinstance(v, Unresolved):
raise v
return to_str(v)
return _TEMPLATE.sub(sub, text)
def evaluate(text, ctx, status='success'):
"""A value that may be a whole ${{ }} expression keeps its type."""
if isinstance(text, str):
m = _TEMPLATE.fullmatch(text.strip())
if m:
v = Expr(ctx, status).eval(m.group(1))
if isinstance(v, Unresolved):
raise v
return v
return interpolate(text, ctx, status)
if isinstance(text, list):
return [evaluate(x, ctx, status) for x in text]
if isinstance(text, dict):
return dict((k, evaluate(v, ctx, status)) for k, v in text.items())
return text
def condition(src, ctx, status):
"""A job's or step's if:, with success() when it has none."""
if src is None:
return status == 'success'
if isinstance(src, bool):
return src
s = str(src).strip()
m = _TEMPLATE.fullmatch(s)
if m:
s = m.group(1)
if not re.search(r'\b(success|failure|always|cancelled)\s*\(', s):
s = 'success() && (%s)' % s
v = Expr(ctx, status).eval(s)
if isinstance(v, Unresolved):
raise v
return truthy(v)
# --------------------------------------------------------------------------
# Workflows
# --------------------------------------------------------------------------
class Workflow:
def __init__(self, path):
self.path = path
self.file = os.path.basename(path)
with open(path, encoding='utf-8') as f:
self.text = f.read()
self.data = yaml_load(self.text) or {}
self.name = self.data.get('name') or os.path.relpath(path, ROOT)
self.jobs = self.data.get('jobs') or {}
def triggers(self):
on = self.data.get(True, self.data.get('on'))
if isinstance(on, str):
return {on: None}
if isinstance(on, list):
return dict((k, None) for k in on)
return on or {}
def inputs(self):
wd = self.triggers().get('workflow_dispatch') or {}
out = {}
for k, v in (wd.get('inputs') or {}).items():
out[k] = (v or {}).get('default')
return out
def load_workflows():
out = []
for name in sorted(os.listdir(WF_DIR)):
if name.endswith(('.yml', '.yaml')):
out.append(Workflow(os.path.join(WF_DIR, name)))
return out
def find_workflow(arg):
wfs = load_workflows()
for wf in wfs:
if arg in (wf.file, os.path.splitext(wf.file)[0], wf.name) \
or os.path.abspath(arg) == wf.path:
return wf
low = arg.lower()
hits = [wf for wf in wfs if low in wf.file.lower() or low in str(wf.name).lower()]
if len(hits) == 1:
return hits[0]
if hits:
sys.exit('ci_local: "%s" matches several workflows: %s'
% (arg, ', '.join(w.file for w in hits)))
sys.exit('ci_local: no workflow named "%s"' % arg)
def git(*args, **kw):
cwd = kw.pop('cwd', ROOT)
return subprocess.run(('git',) + args, cwd=cwd, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
universal_newlines=True).stdout.strip()
def base_context(wf, job_id, workspace, temp):
sha = git('rev-parse', 'HEAD') or '0' * 40
branch = git('rev-parse', '--abbrev-ref', 'HEAD')
ref = 'refs/heads/' + (branch if branch and branch != 'HEAD' else 'master')
repo = 'libretro/RetroArch'
url = git('remote', 'get-url', 'origin')
m = re.search(r'github\.com[:/]+([^/]+/[^/]+?)(?:\.git)?/?$', url or '')
if m:
repo = m.group(1)
github = {
'workflow': wf.name, 'job': job_id, 'sha': sha, 'workflow_sha': sha,
'ref': ref, 'ref_name': ref[len('refs/heads/'):], 'ref_type': 'branch',
'event_name': 'workflow_dispatch', 'event': {}, 'workspace': workspace,
'repository': repo, 'repository_owner': repo.split('/')[0],
'server_url': 'https://github.com', 'actor': 'local', 'token': '',
'run_id': '0', 'run_number': '0', 'run_attempt': '1',
'action_path': '', 'action': '', 'base_ref': '', 'head_ref': '',
}
return {
'github': github,
'env': {},
'vars': {},
'secrets': {},
'inputs': wf.inputs(),
'runner': {'os': host_os(), 'arch': 'X64', 'temp': temp,
'tool_cache': temp, 'name': 'local'},
'matrix': {},
'strategy': {'job-index': 0, 'job-total': 1, 'fail-fast': True,
'max-parallel': 1},
'steps': {},
'needs': {},
'job': {'status': 'success'},
}
def host_os():
if sys.platform.startswith('win') or os.environ.get('MSYSTEM'):
return 'Windows'
if sys.platform == 'darwin':
return 'macOS'
return 'Linux'
def matrix_combos(job, ctx):
strat = job.get('strategy') or {}
m = strat.get('matrix')
if m is None:
return [{}]
m = evaluate(m, ctx)
if not isinstance(m, dict):
raise ExprError('matrix is not a mapping')
include = m.get('include') or []
exclude = m.get('exclude') or []
axes = [(k, v if isinstance(v, list) else [v]) for k, v in m.items()
if k not in ('include', 'exclude')]
combos = [{}]
for k, vals in axes:
combos = [dict(c, **{k: v}) for c in combos for v in vals]
if not axes:
combos = []
def matches(c, e):
return all(k in c and loose_eq(c[k], v) for k, v in e.items())
combos = [c for c in combos if not any(matches(c, e) for e in exclude)]
originals = [dict(c) for c in combos]
for inc in include:
added = False
for c, orig in zip(combos, originals):
if all(k not in orig or loose_eq(orig[k], v) for k, v in inc.items()):
c.update(inc)
added = True
if not added:
combos.append(dict(inc))
originals.append(dict(inc))
return combos or [{}]
def job_order(wf):
order = []
seen = set()
def visit(j, stack):
if j in seen:
return
if j in stack:
raise ExprError('needs: cycle at %s' % j)
needs = wf.jobs[j].get('needs') or []
for n in ([needs] if isinstance(needs, str) else needs):
visit(n, stack + [j])
seen.add(j)
order.append(j)
for j in wf.jobs:
visit(j, [])
return order
STAND_INS = {
'actions/checkout': 'checkout',
'taiki-e/checkout-action': 'checkout',
'actions/setup-python': 'host',
'emscripten-core/setup-emsdk': 'emsdk',
'actions/cache': 'skip',
'actions/cache/restore': 'skip',
'actions/cache/save': 'skip',
'actions/upload-artifact': 'skip',
'actions/download-artifact': 'skip',
}
def action_kind(uses):
return STAND_INS.get(uses.split('@')[0])
def runs_on_os(label):
s = to_str(label).lower()
if 'windows' in s:
return 'Windows'
if 'macos' in s:
return 'macOS'
if 'ubuntu' in s or 'linux' in s:
return 'Linux'
return None
def shell_command(shell, script, plat):
if shell is None:
shell = 'bash' if plat != 'Windows' else 'pwsh'
explicit = False
else:
explicit = True
s = shell.strip()
if s == 'bash':
return (['bash', '--noprofile', '--norc', '-eo', 'pipefail', script]
if explicit else ['bash', '-e', script])
if s == 'sh':
return ['sh', '-e', script]
if s == 'python':
return ['python3', script]
if '{0}' in s and not s.startswith(('msys2', 'pwsh', 'powershell', 'cmd')):
return [script if a == '{0}' else a for a in shlex.split(s)]
return None
# --------------------------------------------------------------------------
# run
# --------------------------------------------------------------------------
class Runner:
def __init__(self, opts):
self.opts = opts
self.failed = []
def say(self, msg):
sys.stdout.write(msg + '\n')
sys.stdout.flush()
def run_workflow(self, wf, workspace):
job_ids = job_order(wf)
if self.opts.job:
if self.opts.job not in wf.jobs:
sys.exit('ci_local: %s has no job "%s" (jobs: %s)'
% (wf.file, self.opts.job, ', '.join(wf.jobs)))
job_ids = [self.opts.job]
results = {}
for jid in job_ids:
results[jid] = self.run_job(wf, jid, workspace, results)
return not self.failed
def run_job(self, wf, jid, workspace, results):
job = wf.jobs[jid]
if 'uses' in job:
self.say('== %s / %s: calls workflow %s, not run' % (wf.file, jid, job['uses']))
return {'result': 'skipped', 'outputs': {}}
temp = tempfile.mkdtemp(prefix='ci_local.')
try:
return self.run_job_in(wf, jid, job, workspace, results, temp)
finally:
if not self.opts.dry_run:
shutil.rmtree(temp, ignore_errors=True)
def run_job_in(self, wf, jid, job, workspace, results, temp):
ctx = base_context(wf, jid, workspace, temp)
ctx['needs'] = dict((k, v) for k, v in results.items())
ctx['env'] = dict((k, to_str(evaluate(v, ctx)))
for k, v in (wf.data.get('env') or {}).items())
combos = matrix_combos(job, ctx)
wanted = self.opts.matrix
picked = [(n, c) for n, c in enumerate(combos)
if all(k in c and loose_eq(c[k], v) for k, v in wanted.items())]
if wanted and not picked:
self.say('== %s / %s: no matrix combination matches %s'
% (wf.file, jid, ' '.join('%s=%s' % kv for kv in wanted.items())))
self.failed.append(jid)
return {'result': 'failure', 'outputs': {}}
outputs = {}
seen = set()
for n, combo in picked:
r, out = self.run_combo(wf, jid, job, workspace, temp, combo, n,
len(combos), results)
outputs.update(out)
seen.add(r)
result = 'failure' if 'failure' in seen else \
'success' if 'success' in seen else 'skipped'
return {'result': result, 'outputs': outputs}
def run_combo(self, wf, jid, job, workspace, temp, combo, index, total,
results):
ctx = base_context(wf, jid, workspace, temp)
ctx['needs'] = results
ctx['matrix'] = combo
ctx['strategy']['job-index'] = index
ctx['strategy']['job-total'] = total
ctx['env'] = dict((k, to_str(evaluate(v, ctx)))
for k, v in (wf.data.get('env') or {}).items())
label = '%s / %s' % (wf.file, jid)
if combo:
label += ' [%s]' % ', '.join('%s=%s' % (k, to_str(v)) for k, v in combo.items())
needs = job.get('needs') or []
needs = [needs] if isinstance(needs, str) else needs
upstream = 'success' if all((results.get(n) or {}).get('result')
in ('success', None) for n in needs) \
else 'failure'
if not condition(job.get('if'), ctx, upstream):
self.say('== %s: skipped (%s)' % (label, 'a job it needs did not pass'
if upstream != 'success' else 'by its if:'))
return 'skipped', {}
plat = runs_on_os(evaluate(job.get('runs-on'), ctx)) or 'Linux'
if plat != host_os() and not self.opts.force:
self.say('== %s: runs on %s, not run here (use --force to try)'
% (label, plat))
return 'skipped', {}
steps = job.get('steps') or []
blocked = [s['uses'] for s in steps if 'uses' in s
and action_kind(s['uses']) is None]
if blocked and not self.opts.force:
self.say('== %s: uses %s, not run here (--force skips those steps)'
% (label, ', '.join(sorted(set(blocked)))))
return 'skipped', {}
container = evaluate(job.get('container'), ctx)
if isinstance(container, str):
container = {'image': container}
engine = None
cid = None
if container and not self.opts.host:
engine = shutil.which('docker') or shutil.which('podman')
if not engine and not self.opts.dry_run:
self.say('== %s: needs container %s; install docker or podman, '
'or pass --host' % (label, container.get('image')))
self.failed.append(label)
return 'failure', {}
for k, v in (job.get('env') or {}).items():
ctx['env'][k] = to_str(evaluate(v, ctx))
defaults = dict(((wf.data.get('defaults') or {}).get('run') or {}))
defaults.update(((job.get('defaults') or {}).get('run') or {}))
self.say('== %s%s' % (label, (' in ' + container['image']) if engine else ''))
files = {}
for name in ('env', 'path', 'output', 'step_summary'):
files[name] = os.path.join(temp, 'github_' + name)
extra_path = []
status = 'success'
if engine and not self.opts.dry_run:
cid = self.start_container(engine, container, workspace, temp)
if not cid:
self.failed.append(label)
self.say('== %s: FAILED' % label)
return 'failure', {}
took = []
try:
for n, step in enumerate(steps):
started = time.time()
status = self.run_step(wf, job, step, n, ctx, status, plat,
defaults, workspace, temp, files,
extra_path, engine, cid)
took.append((time.time() - started,
step.get('name') or step.get('uses') or str(n)))
finally:
if cid:
subprocess.run([engine, 'rm', '-f', cid], stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
outputs = {}
if not self.opts.dry_run:
for k, v in (job.get('outputs') or {}).items():
try:
outputs[k] = interpolate(v, ctx, status)
except Unresolved:
outputs[k] = ''
# where the time went: a slow step is one to make faster
if not self.opts.dry_run:
every = getattr(self.opts, 'times', False)
for secs, name in sorted(took, reverse=True)[:None if every else 3]:
if every or secs >= 10:
self.say(' %4.0f s %s' % (secs, name))
if status == 'failure':
self.failed.append(label)
self.say('== %s: FAILED' % label)
elif not self.opts.dry_run:
self.say('== %s: passed' % label)
return status, outputs
def start_container(self, engine, container, workspace, temp):
args = [engine, 'run', '-d', '--entrypoint', 'tail',
'-v', '%s:%s' % (workspace, workspace),
'-v', '%s:%s' % (temp, temp), '-w', workspace]
args += shlex.split(to_str(container.get('options') or ''))
for k, v in (container.get('env') or {}).items():
args += ['-e', '%s=%s' % (k, to_str(v))]
args += [container['image'], '-f', '/dev/null']
r = subprocess.run(args, stdout=subprocess.PIPE, universal_newlines=True)
if r.returncode:
self.say(' could not start %s' % container['image'])
return None
return r.stdout.strip()
def run_step(self, wf, job, step, n, ctx, status, plat, defaults,
workspace, temp, files, extra_path, engine, cid):
sid = step.get('id') or '__step%d' % n
name = step.get('name') or step.get('uses') or \
(step.get('run') or '').strip().split('\n')[0]
try:
name = interpolate(to_str(name), ctx, status)
except (Unresolved, ExprError):
pass
record = {'outputs': {}, 'outcome': 'skipped', 'conclusion': 'skipped'}
ctx['steps'][sid] = record
for rx in self.opts.skip:
if re.search(rx, name):
self.say('-- %s: skipped (--skip)' % name)
return status
try:
run_it = condition(step.get('if'), ctx, status)
except Unresolved:
if not self.opts.dry_run:
raise
self.say('-- %s: if: %s (decided while the job runs)' % (name, step.get('if')))
run_it = True
if not run_it:
self.say('-- %s: skipped by its if:' % name)
return status
env = dict(ctx['env'])
for k, v in (step.get('env') or {}).items():
try:
env[k] = to_str(evaluate(v, dict(ctx, env=env), status))
except Unresolved:
env[k] = ''
if 'uses' in step:
return self.run_action(step, name, ctx, env, status, workspace, record, extra_path)
script = step.get('run')
if script is None:
return status
try:
body = interpolate(to_str(script), dict(ctx, env=env), status)
except Unresolved:
body = None
shell = step.get('shell') or defaults.get('shell')
wd = step.get('working-directory') or defaults.get('working-directory')
try:
wd = interpolate(to_str(wd), dict(ctx, env=env), status) if wd else ''
except Unresolved:
wd = ''
cwd = os.path.normpath(os.path.join(workspace, wd)) if wd else workspace
if self.opts.dry_run:
self.say('-- %s%s%s' % (name, (' [shell: %s]' % shell) if shell else '',
(' [in %s]' % wd) if wd else ''))
if body is None:
self.say(' (script refers to a value only known while the job runs)')
else:
for line in body.rstrip('\n').split('\n'):
self.say(' | ' + line)
return status
if body is None:
raise ExprError('step "%s" refers to an output that was never set' % name)
self.say('-- %s' % name)
fd, path = tempfile.mkstemp(dir=temp, suffix='.sh' if shell != 'python' else '.py')
with os.fdopen(fd, 'w', encoding='utf-8', newline='\n') as f:
f.write(body)
cmd = shell_command(shell, path.replace(os.sep, '/'), plat)
if cmd is None:
self.say(' shell "%s" is not available here' % shell)
record['outcome'] = record['conclusion'] = 'failure'
return 'failure' if not step.get('continue-on-error') else status
for p in files.values():
open(p, 'a').close()
run_env = dict(os.environ)
# One ccache serves every job here, where on GitHub each job has
# its own: a job's "ccache -M 250M" would cap the one they all
# share, so they evicted each other, and a clean copy is a new
# directory each run, which ccache puts in a debug build's hash, so
# nothing from the last run matched. Not where already set.
for k, v in (('CCACHE_MAXSIZE', '1G'), ('CCACHE_BASEDIR', workspace),
('CCACHE_NOHASHDIR', '1')):
run_env.setdefault(k, v)
run_env.update(env)
gh = ctx['github']
run_env.update({
'CI': 'true', 'GITHUB_ACTIONS': 'true', 'GITHUB_WORKSPACE': workspace,
'GITHUB_SHA': gh['sha'], 'GITHUB_REF': gh['ref'],
'GITHUB_REF_NAME': gh['ref_name'], 'GITHUB_EVENT_NAME': gh['event_name'],
'GITHUB_JOB': gh['job'], 'GITHUB_WORKFLOW': to_str(gh['workflow']),
'GITHUB_REPOSITORY': gh['repository'], 'GITHUB_RUN_ID': '0',
'GITHUB_ENV': files['env'], 'GITHUB_PATH': files['path'],
'GITHUB_OUTPUT': files['output'],
'GITHUB_STEP_SUMMARY': files['step_summary'],
'RUNNER_OS': ctx['runner']['os'], 'RUNNER_TEMP': temp,
'RUNNER_ARCH': 'X64',
})
if extra_path:
run_env['PATH'] = os.pathsep.join(extra_path + [run_env.get('PATH', '')])
open(files['output'], 'w').close()
if cid:
exe = [shutil.which('docker') or shutil.which('podman'), 'exec', '-w', cwd]
for k in env.keys() | set(k for k in run_env if k.startswith(('GITHUB_', 'RUNNER_'))) | {'CI'}:
exe += ['-e', '%s=%s' % (k, run_env[k])]
if extra_path:
cmd = ['sh', '-c', 'PATH="$0:$PATH"; exec "$@"',
':'.join(extra_path)] + cmd
cmd = exe + [cid] + cmd
proc = subprocess.run(cmd)
else:
if not os.path.isdir(cwd):
self.say(' working directory %s does not exist' % wd)
proc = None
else:
proc = subprocess.run(cmd, cwd=cwd, env=run_env)
ok = proc is not None and proc.returncode == 0
self.absorb(files, ctx, record, extra_path)
record['outcome'] = 'success' if ok else 'failure'
if ok or step.get('continue-on-error'):
record['conclusion'] = 'success'
if not ok:
self.say(' failed (continue-on-error)')
return status
record['conclusion'] = 'failure'
self.say(' step failed with exit status %s'
% (proc.returncode if proc is not None else '?'))
return 'failure'
def run_action(self, step, name, ctx, env, status, workspace, record, extra_path=()):
kind = action_kind(step['uses'])
record['outcome'] = record['conclusion'] = 'success'
w = {}
for k, v in (step.get('with') or {}).items():
try:
w[k] = to_str(evaluate(v, dict(ctx, env=env), status))
except Unresolved:
w[k] = ''
if kind == 'checkout' and w.get('repository') \
and w['repository'].lower() != ctx['github']['repository'].lower():
dest = os.path.join(workspace, w.get('path') or '')
if self.opts.dry_run:
self.say('-- %s: clone %s into %s' % (name, w['repository'], dest))
return status
self.say('-- %s: cloning %s' % (name, w['repository']))
if os.path.isdir(os.path.join(dest, '.git')):
return status
args = ['git', 'clone', '--depth', '1']
ref = w.get('ref') or ''
if ref and not re.fullmatch(r'[0-9a-f]{40}', ref):
args += ['--branch', re.sub(r'^refs/(heads|tags)/', '', ref)]
args += ['https://github.com/%s.git' % w['repository'], dest]
if subprocess.run(args).returncode:
record['outcome'] = record['conclusion'] = 'failure'
return 'failure'
return status
if kind == 'emsdk':
version = w.get('version', '')
if self.opts.dry_run:
self.say('-- %s: requires host emcc %s' % (name, version))
return status
run_env = dict(os.environ)
run_env.update(env)
if extra_path:
run_env['PATH'] = os.pathsep.join(list(extra_path) + [run_env.get('PATH', '')])
command = shutil.which('emcc', path=run_env.get('PATH'))
result = None
if command and re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+', version):
try:
result = subprocess.run([command, '--version'], cwd=workspace,
env=run_env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True, timeout=30)
except (OSError, subprocess.TimeoutExpired):
pass
line = result.stdout.splitlines()[0] if result and result.stdout else ''
found = re.search(r'\)\s+([0-9]+\.[0-9]+\.[0-9]+)(?:\s|$)', line)
if not result or result.returncode or not line.startswith('emcc ') \
or not found or found.group(1) != version:
self.say('-- %s: install the pinned host emcc %s before running this job'
% (name, version))
record['outcome'] = record['conclusion'] = 'failure'
return 'failure'
self.say('-- %s: host emcc %s verified' % (name, version))
return status
if kind in ('checkout', 'host'):
self.say('-- %s: this checkout / host tools stand in' % name)
elif kind == 'skip':
self.say('-- %s: skipped (no local equivalent needed)' % name)
else:
self.say('-- %s: action not run (--force)' % name)
return status
def absorb(self, files, ctx, record, extra_path):
for k, v in parse_env_file(files['env']).items():
ctx['env'][k] = v
open(files['env'], 'w').close()
record['outputs'].update(parse_env_file(files['output']))
with open(files['path'], encoding='utf-8', errors='replace') as f:
for line in f:
line = line.strip()
if line and line not in extra_path:
extra_path.insert(0, line)
open(files['path'], 'w').close()
def parse_env_file(path):
out = {}
try:
with open(path, encoding='utf-8', errors='replace') as f:
lines = f.read().split('\n')
except OSError:
return out
i = 0
while i < len(lines):
line = lines[i]
m = re.match(r'^([^=<]+)<<(.+)$', line)
if m:
key, delim = m.group(1), m.group(2)
val = []
i += 1
while i < len(lines) and lines[i] != delim:
val.append(lines[i])
i += 1
out[key] = '\n'.join(val)
elif '=' in line:
k, v = line.split('=', 1)
out[k] = v
i += 1
return out
def clean_copy():
dest = tempfile.mkdtemp(prefix='ci_local.tree.')
os.rmdir(dest)
r = subprocess.run(['git', 'worktree', 'add', '--detach', dest, 'HEAD'],
cwd=ROOT, stdout=subprocess.DEVNULL)
if r.returncode:
sys.exit('ci_local: git worktree add failed')
diff = subprocess.run(['git', 'diff', 'HEAD', '--binary'], cwd=ROOT,
stdout=subprocess.PIPE).stdout
if diff:
subprocess.run(['git', 'apply', '--binary', '-'], cwd=dest, input=diff,
check=True)
others = subprocess.run(['git', 'ls-files', '-z', '--others',
'--exclude-standard'], cwd=ROOT,
stdout=subprocess.PIPE).stdout.split(b'\0')
for rel in others:
if not rel:
continue
rel = rel.decode('utf-8', 'surrogateescape')
src = os.path.join(ROOT, rel)
dst = os.path.join(dest, rel)
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copy2(src, dst)
return dest
# --------------------------------------------------------------------------
# which
# --------------------------------------------------------------------------
SRC_EXT = ('.c', '.cpp', '.cc', '.cxx', '.m', '.mm', '.h', '.hpp', '.S', '.s',
'.glsl', '.slang', '.inc', '.def')
_PATHISH = re.compile(r'[A-Za-z0-9_.$(){}/+-]*[A-Za-z0-9_]/[A-Za-z0-9_.$(){}/+-]*'
r'|[A-Za-z0-9_+-]+\.(?:c|cpp|cc|m|mm|h|py|sh|mk|list|ref)\b')
_TREE_BUILD = re.compile(r'(?:^|[\s;&|(])(?:\./configure|\./bootstrap'
r'|make\b[^\n]*-f\s*Makefile\.[\w.]+|griffin'
r'|msbuild|cmake\b|xcodebuild|qb/)', re.M)
_CHECKER = re.compile(r'(?:^|[\s;&|(/])tools/[\w.-]+\.(?:py|sh)\b', re.M)
def _strip_vars(tok):
tok = re.sub(r'\$[({][A-Za-z0-9_]+[)}]/?', '', tok)
tok = re.sub(r'\$[A-Za-z_][A-Za-z0-9_]*/?', '', tok)
while tok.startswith(('../', './')):
tok = tok[3:] if tok.startswith('../') else tok[2:]
return tok.strip('/')
class Coverage:
"""What a job reads: files and sample directories it names, and the
sources the Makefiles in those directories list."""
def __init__(self):
self.files = {}
self.dirs = {}
self.suffixes = {}
def add_text(self, text, base, via, depth=0):
for tok in set(_PATHISH.findall(text)):
t = tok.strip('"\'(),;:')
if not t or t.startswith(('http', '-', '$')) and '/' not in t:
continue
for b in (ROOT, base):
p = os.path.normpath(os.path.join(b, _strip_vars(t) if t.startswith('$') else t))
if not p.startswith(ROOT + os.sep):
continue
rel = os.path.relpath(p, ROOT).replace(os.sep, '/')
if os.path.isfile(p):
if rel not in self.files:
self.files[rel] = via
if depth < 2 and rel.endswith(('.sh', '.py')) \
and not rel.startswith('.github/workflows/') or \
depth < 2 and rel.startswith('.github/workflows/scripts/'):
self.add_file_text(p, via + ' -> ' + rel, depth + 1)
break
if os.path.isdir(p) and rel.count('/') >= 1:
if rel not in self.dirs:
self.dirs[rel] = via
self.add_makefiles(p, rel, via)
break
def add_file_text(self, path, via, depth):
try:
with open(path, encoding='utf-8', errors='replace') as f:
text = f.read()
except OSError:
return
self.add_text(text, os.path.dirname(path), via, depth)
def add_makefiles(self, path, rel, via):
for name in sorted(os.listdir(path)):
if not (name.startswith('Makefile') or name.endswith('.mk')
or name == 'CMakeLists.txt'):
continue
mk = os.path.join(path, name)
if not os.path.isfile(mk):
continue
with open(mk, encoding='utf-8', errors='replace') as f:
text = f.read()
for tok in re.findall(r'[A-Za-z0-9_.$(){}/+-]+\.(?:c|cpp|cc|m|mm|S)\b', text):
s = _strip_vars(tok)
if s:
self.suffixes.setdefault(s, '%s -> %s/%s' % (via, rel, name))
def covers(self, rel):
if rel in self.files:
return self.files[rel]
for d, via in self.dirs.items():
if rel.startswith(d + '/'):
return '%s (in %s)' % (via, d)
for s, via in self.suffixes.items():
if rel == s or rel.endswith('/' + s):
return '%s lists %s' % (via, s)
return None
def job_texts(job):
for step in job.get('steps') or []:
if 'run' in step:
yield step, to_str(step.get('run')), to_str(step.get('working-directory') or '')
def paths_filter_hit(wf, rel):
# A filter tools/ci_paths.py wrote takes in everything but other
# samples: the coverage below says more about what the jobs read
if '# ci_paths: generated by tools/ci_paths.py' in wf.text:
return None
trig = wf.triggers()
for ev in ('push', 'pull_request'):
spec = trig.get(ev) or {}
if not isinstance(spec, dict) or not spec.get('paths'):
continue
for pat in spec['paths']:
pat = to_str(pat)
if pat.startswith('!'):
continue
rx = re.escape(pat).replace(r'\*\*', '.*').replace(r'\*', '[^/]*')
if re.fullmatch(rx, rel) or fnmatch.fnmatch(rel, pat):
return pat
return None
def cmd_which(paths):
rels = []
for p in paths:
a = os.path.abspath(p)
if not a.startswith(ROOT):
a = os.path.join(ROOT, p)
rels.append(os.path.relpath(a, ROOT).replace(os.sep, '/'))
direct, builds, checkers = [], [], []
for wf in load_workflows():
for jid, job in wf.jobs.items():
if not isinstance(job, dict) or 'steps' not in job:
continue
cov = Coverage()
texts = list(job_texts(job))
for step, text, wd in texts:
base = os.path.join(ROOT, wd) if wd and '${{' not in wd else ROOT
cov.add_text(text + '\n' + wd, base, 'named in a run: step')
plat = runs_on_os(job.get('runs-on')) or ('matrix' if '${{' in to_str(job.get('runs-on')) else '?')
tag = '%s --job %s' % (wf.file, jid)
hits = []
for rel in rels:
if rel == '.github/workflows/' + wf.file:
hits.append('%s: is this workflow' % rel)
continue
pat = paths_filter_hit(wf, rel)
if pat:
hits.append('%s: paths filter %s' % (rel, pat))
continue
why = cov.covers(rel)
if why:
hits.append('%s: %s' % (rel, why))
alltext = '\n'.join(t for _, t, _ in texts)
if hits:
direct.append((tag, plat, hits))
elif any(r.endswith(SRC_EXT) for r in rels) and _TREE_BUILD.search(alltext):
builds.append((tag, plat))
elif _CHECKER.search(alltext) and not re.search(r'(?:^|\s)(?:make|gcc|cc|clang)\b', alltext, re.M):
checkers.append((tag, plat))
def show(title, rows):
if not rows:
return
print(title)
for row in rows:
print(' %-58s %s' % (row[0], row[1]))
for h in (row[2] if len(row) > 2 else []):
print(' ' + h)
print()
show('Jobs that name these files (run all of them):', direct)
show('Full-tree builds (run those for the platforms you can):', builds)
show('Tree-wide checks driven by tools/ scripts (run all of them):', checkers)
print('Run one with: tools/ci_local.py run <workflow> --job <job>')
def cmd_list():
for wf in load_workflows():
print('%s (%s)' % (wf.file, wf.name))
for jid, job in wf.jobs.items():
if not isinstance(job, dict):
continue
ctx = base_context(wf, jid, ROOT, tempfile.gettempdir())
try:
n = len(matrix_combos(job, ctx))
except (ExprError, Unresolved):
n = '?'
ro = to_str(job.get('runs-on'))
extra = ' container' if job.get('container') else ''
print(' %-40s %-28s matrix %s%s' % (jid, ro, n, extra))
# --------------------------------------------------------------------------
# selftest
# --------------------------------------------------------------------------
def _same(a, b):
if isinstance(a, float) and isinstance(b, float) and a != a and b != b:
return True
if type(a) is not type(b):
return False
if isinstance(a, dict):
return list(a.keys()) == list(b.keys()) and all(_same(a[k], b[k]) for k in a)
if isinstance(a, list):
return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b))
return a == b
def _first_diff(a, b, path='$'):
if isinstance(a, dict) and isinstance(b, dict):
for k in list(a) + [k for k in b if k not in a]:
if k not in a or k not in b:
return '%s.%s present on one side only' % (path, k)
d = _first_diff(a[k], b[k], '%s.%s' % (path, k))
if d:
return d
return None
if isinstance(a, list) and isinstance(b, list):
for n, (x, y) in enumerate(zip(a, b)):
d = _first_diff(x, y, '%s[%d]' % (path, n))
if d:
return d
if len(a) != len(b):
return '%s length %d != %d' % (path, len(a), len(b))
return None
if not _same(a, b):
return '%s: %r != %r' % (path, a, b)
return None
YAML_CASES = [
('a: 1\nb: [x, "y z", 3]\nc: { d: on, e: 10.4 }\n', None),
('k: |\n one\n two\n\n three\n\nn: x\n', None),
('k: |-\n a\n b\n', None),
('s:\n- a\n- b: c\n d: e\n-\n - f\n', None),
("q: 'it''s'\nr: \"a\\tb\"\nu: plain # comment\n", None),
('m: >\n folded\n text\n\n para\n', None),
('x: "multi\n line"\ny: 0o7\nz: 017\nw: ~\n', None),
]
EXPR_CASES = [
("'a' == 'A'", True),
("1 == '1'", True),
("null == 0", True),
("matrix.x == ''", True),
("startsWith(matrix.name, 'aarch64') || matrix.name == 'armv7'", True),
("github.event.pull_request.number || github.ref", 'refs/heads/master'),
("github.event_name == 'release' && '0' || '1'", '1'),
("matrix.n == '' && 15 || 40", 40.0),
("!(1 == 2) && contains('abc', 'B')", True),
("format('{0}-{1}', 'a', 2)", 'a-2'),
("fromJSON('[1,2]')[1]", 2),
("strategy.job-index", 0),
]
FIXTURE = """\
name: fixture
on: [workflow_dispatch]
env:
TOP: top-${{ github.event_name }}
jobs:
first:
runs-on: ubuntu-latest
outputs:
word: ${{ steps.make.outputs.word }}
strategy:
matrix:
a: [x, y]
include:
- a: y
extra: more
steps:
- uses: actions/checkout@v4
- id: make
run: |
echo "word=hello-${{ matrix.a }}" >> "$GITHUB_OUTPUT"
echo "FROMENV=set" >> "$GITHUB_ENV"
mkdir -p "$RUNNER_TEMP/bin"
printf '#!/bin/sh\\necho on-path\\n' > "$RUNNER_TEMP/bin/fixture_tool"
chmod +x "$RUNNER_TEMP/bin/fixture_tool"
echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
mkdir -p sub
- shell: bash
working-directory: sub
run: |
test "$FROMENV" = set
test "$TOP" = top-workflow_dispatch
test "$(basename "$PWD")" = sub
test "$(fixture_tool)" = on-path
test "${{ matrix.a }}" = x || test "${{ matrix.extra }}" = more
- if: steps.make.outputs.word == 'hello-y'
run: touch "$GITHUB_WORKSPACE/only_y"
- continue-on-error: true
run: exit 3
second:
needs: first
runs-on: ubuntu-latest
steps:
- run: test "${{ needs.first.outputs.word }}" = hello-y
third:
needs: second
runs-on: ubuntu-latest
steps:
- shell: bash
run: false | true
fourth:
needs: third
runs-on: ubuntu-latest
steps:
- run: touch "$GITHUB_WORKSPACE/must_not_exist"
"""
def _run_fixture():
"""Runs FIXTURE on this machine: environment, path and output files,
matrix include, needs, continue-on-error and bash's pipefail."""
ws = tempfile.mkdtemp(prefix='ci_local.fixture.')
try:
path = os.path.join(ws, 'fixture.yml')
with open(path, 'w', encoding='utf-8') as f:
f.write(FIXTURE)
wf = Workflow(path)
class Opts:
job = None
matrix = {}
skip = []
dry_run = False
host = True
force = True
out = []
r = Runner(Opts())
r.say = out.append
ok = r.run_workflow(wf, ws)
want_failed = ['fixture.yml / third']
problems = []
if ok or r.failed != want_failed:
problems.append('failed jobs %r, expected %r' % (r.failed, want_failed))
if not os.path.exists(os.path.join(ws, 'only_y')):
problems.append('the matrix y step did not run')
if os.path.exists(os.path.join(ws, 'must_not_exist')):
problems.append('a job ran after the job it needs failed')
for p in problems:
print('FAIL run fixture: ' + p)
if problems:
print('\n'.join(' ' + l for l in out))
return len(problems)
finally:
shutil.rmtree(ws, ignore_errors=True)
def _emsdk_fixture():
from unittest import mock
class Opts:
dry_run = False
runner = Runner(Opts())
runner.say = lambda msg: None
step = {'uses': 'emscripten-core/setup-emsdk@v15',
'with': {'version': '${{ env.EM_VERSION }}'}}
failures = 0
for tool, requested, code, output, expected in [
('emcc', '5.0.1', 0, 'emcc (Emscripten) 5.0.1 (hash)\n', 'success'),
('emcc', '5.0.1', 0, 'emcc (Emscripten) 5.0.10 (hash)\n', 'failure'),
('emcc', '5.0.1', 1, 'emcc (Emscripten) 5.0.1\n', 'failure'),
('emcc', '5.0.1', 0, '', 'failure'),
('emcc', '5.0.1', 0, 'clang (LLVM) 5.0.1\n', 'failure'),
(None, '5.0.1', 0, '', 'failure'),
('emcc', 'latest', 0, '', 'failure')]:
ctx = {'env': {'EM_VERSION': requested}, 'github': {'repository': 'test/repo'}}
record = {}
result = subprocess.CompletedProcess([], code, stdout=output)
with mock.patch.dict(os.environ, {'EM_CONFIG': 'fixture-sdk'}), \
mock.patch.object(shutil, 'which', return_value=tool), \
mock.patch.object(subprocess, 'run', return_value=result) as run:
status = runner.run_action(step, 'SDK', ctx, ctx['env'], 'success', ROOT, record)
failures += status != expected or record.get('conclusion') != expected
if tool and requested == '5.0.1':
failures += run.call_count != 1
failures += run.call_args[1]['env'].get('EM_CONFIG') != 'fixture-sdk'
failures += run.call_args[1]['env'].get('EM_VERSION') != requested
else:
failures += run.call_count != 0
ctx['env']['EM_VERSION'] = '5.0.1'
for error in (OSError('not executable'), subprocess.TimeoutExpired(['emcc'], 30)):
with mock.patch.object(shutil, 'which', return_value='emcc'), \
mock.patch.object(subprocess, 'run', side_effect=error) as run:
record = {}
status = runner.run_action(step, 'SDK', ctx, ctx['env'], 'success', ROOT, record)
failures += status != 'failure' or record.get('conclusion') != 'failure'
failures += run.call_count != 1
ctx['env']['PATH'] = 'workflow-path'
with mock.patch.object(shutil, 'which', return_value='emcc') as which, \
mock.patch.object(subprocess, 'run', return_value=
subprocess.CompletedProcess([], 0, stdout='emcc (SDK) 5.0.1\n')) as run:
record = {}
status = runner.run_action(step, 'SDK', ctx, ctx['env'], 'success', ROOT,
record, ['ci-sdk'])
expected_path = os.pathsep.join(['ci-sdk', 'workflow-path'])
failures += status != 'success'
failures += which.call_args[1].get('path') != expected_path
failures += run.call_args[1]['env'].get('PATH') != expected_path
runner.opts.dry_run = True
with mock.patch.object(subprocess, 'run') as run:
record = {}
runner.run_action(step, 'SDK', ctx, ctx['env'], 'success', ROOT, record)
failures += run.call_count != 0
return failures
def cmd_selftest():
bad = _emsdk_fixture()
try:
import yaml
except ImportError:
yaml = None
print('PyYAML is not installed: parser not compared against it')
for text, _ in YAML_CASES:
if yaml:
want = yaml.safe_load(text)
got = yaml_load(text)
if not _same(got, want):
print('FAIL yaml case %r: %s' % (text, _first_diff(got, want)))
bad += 1
wfs = []
for name in sorted(os.listdir(WF_DIR)):
if not name.endswith(('.yml', '.yaml')):
continue
path = os.path.join(WF_DIR, name)
try:
wf = Workflow(path)
except Exception as e:
print('FAIL %s: does not parse: %s' % (name, e))
bad += 1
continue
wfs.append(wf)
if yaml:
with open(path, encoding='utf-8') as f:
want = yaml.safe_load(f)
if not _same(wf.data, want):
print('FAIL %s: parses differently from PyYAML: %s'
% (name, _first_diff(wf.data, want)))
bad += 1
ctx = {'matrix': {'x': '', 'name': 'aarch64-linux', 'n': 'x'},
'github': {'event': {}, 'ref': 'refs/heads/master',
'event_name': 'workflow_dispatch'},
'strategy': {'job-index': 0}}
for src, want in EXPR_CASES:
try:
got = Expr(ctx, 'success').eval(src)
except Exception as e:
got = e
if not (got == want and type(got) is type(want)):
print('FAIL expression %s: %r, expected %r' % (src, got, want))
bad += 1
class Opts:
job = None
matrix = {}
skip = []
dry_run = True
host = False
force = True
devnull = open(os.devnull, 'w')
for wf in wfs:
r = Runner(Opts())
r.say = lambda msg: devnull.write(msg)
try:
r.run_workflow(wf, ROOT)
except Exception as e:
print('FAIL %s: dry run: %s: %s' % (wf.file, type(e).__name__, e))
bad += 1
devnull.close()
if shutil.which('bash'):
bad += _run_fixture()
else:
print('bash is not on PATH: run fixture skipped')
print('%d workflows, %s' % (len(wfs), 'ok' if not bad else '%d failures' % bad))
return 1 if bad else 0
# --------------------------------------------------------------------------
def main(argv):
if not argv or argv[0] in ('-h', '--help', 'help'):
print(__doc__.strip())
return 0
cmd, rest = argv[0], argv[1:]
if cmd == 'list':
cmd_list()
return 0
if cmd == 'which':
if not rest:
sys.exit('ci_local: which needs at least one path')
cmd_which(rest)
return 0
if cmd == 'selftest':
return cmd_selftest()
if cmd != 'run':
sys.exit('ci_local: unknown command "%s" (list, which, run, selftest)' % cmd)
class Opts:
job = None
matrix = {}
skip = []
dry_run = False
clean = False
host = False
force = False
times = False
opts = Opts()
opts.matrix = {}
opts.skip = []
target = None
i = 0
while i < len(rest):
a = rest[i]
if a in ('--job', '--matrix', '--skip'):
if i + 1 >= len(rest):
sys.exit('ci_local: %s needs a value' % a)
v = rest[i + 1]
i += 2
if a == '--job':
opts.job = v
elif a == '--skip':
opts.skip.append(v)
else:
if '=' not in v:
sys.exit('ci_local: --matrix takes KEY=VALUE')
k, val = v.split('=', 1)
opts.matrix[k] = val
continue
if a == '--dry-run':
opts.dry_run = True
elif a == '--clean-copy':
opts.clean = True
elif a == '--host':
opts.host = True
elif a == '--force':
opts.force = True
elif a == '--times':
opts.times = True
elif a.startswith('-'):
sys.exit('ci_local: unknown option %s' % a)
elif target is None:
target = a
else:
sys.exit('ci_local: one workflow at a time')
i += 1
if target is None:
sys.exit('ci_local: run needs a workflow')
wf = find_workflow(target)
workspace = ROOT
if opts.clean and not opts.dry_run:
workspace = clean_copy()
print('ci_local: running in %s' % workspace)
try:
ok = Runner(opts).run_workflow(wf, workspace)
finally:
if workspace != ROOT:
subprocess.run(['git', 'worktree', 'remove', '--force', workspace],
cwd=ROOT)
return 0 if ok else 1
if __name__ == '__main__':
if hasattr(__import__('signal'), 'SIGPIPE'):
import signal
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
sys.exit(main(sys.argv[1:]))