xenia/xenia-build
2015-08-18 14:18:00 -07:00

1014 lines
27 KiB
Python
Executable File

#!/usr/bin/env python
# Copyright 2015 Ben Vanik. All Rights Reserved.
"""Main build script and tooling for xenia.
Run with --help or no arguments for possible commands.
"""
__author__ = 'ben.vanik@gmail.com (Ben Vanik)'
import argparse
import os
import re
import shutil
import string
import subprocess
import sys
self_path = os.path.dirname(os.path.abspath(__file__))
def main():
# Add self to the root search path.
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
# Augment path to include our fancy things.
os.environ['PATH'] += os.pathsep + os.pathsep.join([
self_path,
os.path.abspath('build_tools/'),
])
# Check git exists.
if not has_bin('git'):
print('ERROR: git must be installed and on PATH.')
sys.exit(1)
return
# Check python version.
if not sys.version_info[:2] == (2, 7):
# TODO(benvanik): allow things to work with 3, but warn on clang-format.
print('ERROR: Python 2.7 must be installed and on PATH')
sys.exit(1)
return
# Grab Visual Studio version and execute shell to set up environment.
if sys.platform == 'win32':
vs_version = import_vs_environment()
if vs_version != 2015:
print('ERROR: Visual Studio 2015 not found!')
print('Ensure you have the VS140COMNTOOLS environment variable!')
sys.exit(1)
return
# Setup main argument parser and common arguments.
parser = argparse.ArgumentParser(prog='xenia-build')
# Grab all commands and populate the argument parser for each.
subparsers = parser.add_subparsers(title='subcommands',
dest='subcommand')
commands = discover_commands(subparsers)
# If the user passed no args, die nicely.
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
return
# Gather any arguments that we want to pass to child processes.
command_args = sys.argv[1:]
pass_args = []
try:
pass_index = command_args.index('--')
pass_args = command_args[pass_index + 1:]
command_args = command_args[:pass_index]
except:
pass
# Parse command name and dispatch.
args = vars(parser.parse_args(command_args))
command_name = args['subcommand']
try:
command = commands[command_name]
return_code = command.execute(args, pass_args, os.getcwd())
except Exception as e:
raise
return_code = 1
sys.exit(return_code)
# TODO(benvanik): move to build_tools utils module.
def import_vs_environment():
"""Finds the installed Visual Studio version and imports
interesting environment variables into os.environ.
Returns:
A version such as 2015 or None if no VS is found.
"""
version = 0
tools_path = ''
if 'VS140COMNTOOLS' in os.environ:
version = 2015
tools_path = os.environ['VS140COMNTOOLS']
if version == 0:
return None
tools_path = os.path.join(tools_path, '..\\..\\vc\\vcvarsall.bat')
args = [tools_path, '&&', 'set']
popen = subprocess.Popen(
args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
variables, _ = popen.communicate()
envvars_to_save = (
'devenvdir',
'include',
'lib',
'libpath',
'path',
'pathext',
'systemroot',
'temp',
'tmp',
'windowssdkdir',
)
for line in variables.splitlines():
for envvar in envvars_to_save:
if re.match(envvar + '=', line.lower()):
var, setting = line.split('=', 1)
if envvar == 'path':
setting = os.path.dirname(sys.executable) + os.pathsep + setting
os.environ[var.upper()] = setting
break
os.environ['VSVERSION'] = str(version)
return version
def has_bin(bin):
"""Checks whether the given binary is present.
"""
for path in os.environ["PATH"].split(os.pathsep):
path = path.strip('"')
exe_file = os.path.join(path, bin)
if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
return True
exe_file = exe_file + '.exe'
if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
return True
return None
def shell_call(command, throw_on_error=True, stdout_path=None):
"""Executes a shell command.
Args:
command: Command to execute, as a list of parameters.
throw_on_error: Whether to throw an error or return the status code.
stdout_path: File path to write stdout output to.
Returns:
If throw_on_error is False the status code of the call will be returned.
"""
stdout_file = None
if stdout_path:
stdout_file = open(stdout_path, 'w')
result = 0
try:
if throw_on_error:
result = 1
subprocess.check_call(command, shell=False, stdout=stdout_file)
result = 0
else:
result = subprocess.call(command, shell=False, stdout=stdout_file)
finally:
if stdout_file:
stdout_file.close()
return result
def git_submodule_update():
"""Runs a full recursive git submodule init and update.
Older versions of git do not support 'update --init --recursive'. We could
check and run it on versions that do support it and speed things up a bit.
"""
if True:
shell_call([
'git',
'submodule',
'update',
'--init',
'--recursive',
])
else:
shell_call([
'git',
'submodule',
'init',
])
shell_call([
'git',
'submodule',
'foreach',
'--recursive',
'git',
'submodule',
'init',
])
shell_call([
'git',
'submodule',
'update',
'--recursive',
])
def get_clang_format_binary():
"""Finds a clang-format binary. Aborts if none is found.
Returns:
A path to the clang-format executable.
"""
attempts = [
'C:\\Program Files (x86)\\LLVM\\bin\\clang-format.exe',
'clang-format-3.8',
'clang-format',
]
for binary in attempts:
if has_bin(binary):
return binary
print 'ERROR: clang-format is not on PATH'
print 'LLVM is available from http://llvm.org/releases/download.html'
print 'At least version 3.8 is required.'
print 'See docs/style_guide.md for instructions on how to get it.'
sys.exit(1)
def run_premake(target_os, action):
"""Runs premake on the main project with the given format.
Args:
target_os: target --os to pass to premake.
action: action to preform.
"""
shell_call([
'python',
os.path.join('build_tools', 'premake'),
'--file=premake5.lua',
'--os=%s' % (target_os),
'--cc=clang',
'--test-suite-mode=combined',
'--verbose',
action,
])
def run_premake_clean():
"""Runs a premake clean operation.
"""
if sys.platform == 'darwin':
run_premake('macosx', 'clean')
elif sys.platform == 'win32':
run_premake('windows', 'clean')
else:
run_premake('linux', 'clean')
def run_platform_premake():
"""Runs all gyp configurations.
"""
if sys.platform == 'darwin':
run_premake('macosx', 'xcode')
elif sys.platform == 'win32':
run_premake('windows', 'vs2015')
else:
run_premake('linux', 'gmake')
run_premake('linux', 'codelite')
def get_build_bin_path(args):
"""Returns the path of the bin/ path with build results based on the
configuration specified in the parsed arguments.
Args:
args: Parsed arguments.
Returns:
A full path for the bin folder.
"""
if sys.platform == 'darwin':
platform = 'macosx'
elif sys.platform == 'win32':
platform = 'windows'
else:
platform = 'linux'
return os.path.join(self_path, 'build', 'bin', platform, args['config'])
def discover_commands(subparsers):
"""Looks for all commands and returns a dictionary of them.
In the future commands could be discovered on disk.
Args:
subparsers: Argument subparsers parent used to add command parsers.
Returns:
A dictionary containing name-to-Command mappings.
"""
commands = {
'setup': SetupCommand(subparsers),
'pull': PullCommand(subparsers),
'premake': PremakeCommand(subparsers),
'build': BuildCommand(subparsers),
'gentests': GenTestsCommand(subparsers),
'test': TestCommand(subparsers),
'clean': CleanCommand(subparsers),
'nuke': NukeCommand(subparsers),
'lint': LintCommand(subparsers),
'format': FormatCommand(subparsers),
'style': StyleCommand(subparsers),
}
if sys.platform == 'win32':
commands['devenv'] = DevenvCommand(subparsers)
return commands
class Command(object):
"""Base type for commands.
"""
def __init__(self, subparsers, name, help_short=None, help_long=None,
*args, **kwargs):
"""Initializes a command.
Args:
subparsers: Argument subparsers parent used to add command parsers.
name: The name of the command exposed to the management script.
help_short: Help text printed alongside the command when queried.
help_long: Extended help text when viewing command help.
"""
self.name = name
self.help_short = help_short
self.help_long = help_long
self.parser = subparsers.add_parser(name,
help=help_short,
description=help_long)
self.parser.set_defaults(command_handler=self)
def execute(self, args, pass_args, cwd):
"""Executes the command.
Args:
args: Arguments hash for the command.
pass_args: Arguments list to pass to child commands.
cwd: Current working directory.
Returns:
Return code of the command.
"""
return 1
class SetupCommand(Command):
"""'setup' command."""
def __init__(self, subparsers, *args, **kwargs):
super(SetupCommand, self).__init__(
subparsers,
name='setup',
help_short='Setup the build environment.',
*args, **kwargs)
def execute(self, args, pass_args, cwd):
print('Setting up the build environment...')
print('')
# Setup submodules.
print('- git submodule init / update...')
git_submodule_update()
print('')
print('- running premake...')
run_platform_premake()
print('')
print('Success!')
return 0
class PullCommand(Command):
"""'pull' command."""
def __init__(self, subparsers, *args, **kwargs):
super(PullCommand, self).__init__(
subparsers,
name='pull',
help_short='Pulls the repo and all dependencies and rebases changes.',
*args, **kwargs)
self.parser.add_argument('--merge', action='store_true',
help='Merges on master instead of rebasing.')
def execute(self, args, pass_args, cwd):
print('Pulling...')
print('')
print('- switching to master...')
shell_call([
'git',
'checkout',
'master',
])
print('')
print('- pulling self...')
if args['merge']:
shell_call([
'git',
'pull',
])
else:
shell_call([
'git',
'pull',
'--rebase',
])
print('')
print('- pulling dependencies...')
git_submodule_update()
print('')
print('- running premake...')
run_platform_premake()
print('')
print('Success!')
return 0
class PremakeCommand(Command):
"""'premake' command."""
def __init__(self, subparsers, *args, **kwargs):
super(PremakeCommand, self).__init__(
subparsers,
name='premake',
help_short='Runs premake to update all projects.',
*args, **kwargs)
def execute(self, args, pass_args, cwd):
print('Running premake...')
print('')
# Update premake.
run_platform_premake()
print('Success!')
return 0
class BaseBuildCommand(Command):
"""Base command for things that require building."""
def __init__(self, subparsers, *args, **kwargs):
super(BaseBuildCommand, self).__init__(
subparsers,
*args, **kwargs)
self.parser.add_argument(
'--config', choices=['checked', 'debug', 'release'], default='debug',
help='Chooses the build configuration.')
self.parser.add_argument(
'--target', action='append', default=[],
help='Builds only the given target(s).')
self.parser.add_argument(
'--force', action='store_true',
help='Forces a full rebuild.')
self.parser.add_argument(
'--no-premake', action='store_true',
help='Skips running premake before building.')
def execute(self, args, pass_args, cwd):
if not args['no_premake']:
print('- running premake...')
run_platform_premake()
print('')
print('- building (%s):%s...' % (
'all' if not len(args['target']) else ' '.join(args['target']),
args['config']))
if sys.platform == 'win32':
result = shell_call([
'devenv',
'/nologo',
'build/xenia.sln',
'/rebuild' if args['force'] else '/build',
args['config'],
] + [('/project ', target) for target in args['target']] +
pass_args, throw_on_error=False)
elif sys.platform == 'darwin':
# TODO(benvanik): other platforms.
print('ERROR: don\'t know how to build on this platform.')
else:
# TODO(benvanik): allow gcc?
os.environ['CXX'] = 'clang++-3.8'
os.environ['CC'] = 'clang++-3.8'
result = shell_call([
'make',
'-Cbuild/',
'config=%s_linux' % (args['config']),
] + pass_args + args['target'], throw_on_error=False)
print('')
if result != 0:
print('ERROR: build failed with one or more errors.')
return result
return 0
class BuildCommand(BaseBuildCommand):
"""'build' command."""
def __init__(self, subparsers, *args, **kwargs):
super(BuildCommand, self).__init__(
subparsers,
name='build',
help_short='Builds the project.',
*args, **kwargs)
def execute(self, args, pass_args, cwd):
print('Building %s...' % (args['config']))
print('')
result = super(BuildCommand, self).execute(args, pass_args, cwd)
if not result:
print('Success!')
return result
class TestCommand(BaseBuildCommand):
"""'test' command."""
def __init__(self, subparsers, *args, **kwargs):
super(TestCommand, self).__init__(
subparsers,
name='test',
help_short='Runs automated tests that have been built with `xb build`.',
help_long='''
To pass arguments to the test executables separate them with `--`.
For example, you can run only the instr_foo.s tests with:
$ xb test -- instr_foo
''',
*args, **kwargs)
self.parser.add_argument(
'--no-build', action='store_true',
help='Don\'t build before running tests.')
self.parser.add_argument(
'--continue', action='store_true',
help='Don\'t stop when a test errors, but continue running all.')
def execute(self, args, pass_args, cwd):
print('Testing...')
print('')
# The test executables that will be built and run.
test_targets = args['target'] or [
'xenia-cpu-frontend-tests',
]
args['target'] = test_targets
# Build all targets (if desired).
if not args['no_build']:
result = super(TestCommand, self).execute(args, [], cwd)
if result:
print('Failed to build, aborting test run.')
return result
# Ensure all targets exist before we run.
test_executables = [
os.path.join(get_build_bin_path(args), test_target)
for test_target in test_targets]
for test_executable in test_executables:
if not has_bin(test_executable):
print('ERROR: Unable to find %s - build it.' % (test_executable))
return 1
# Run tests.
any_failed = False
for test_executable in test_executables:
print('- %s' % (test_executable))
result = shell_call([
test_executable,
] + pass_args,
throw_on_error=False)
if result:
any_failed = True
if args['continue']:
print('ERROR: test failed but continuing due to --continue.')
else:
print('ERROR: test failed, aborting, use --continue to keep going.')
return result
if any_failed:
print('ERROR: one or more tests failed.')
result = 1
return result
class GenTestsCommand(Command):
"""'gentests' command."""
def __init__(self, subparsers, *args, **kwargs):
super(GenTestsCommand, self).__init__(
subparsers,
name='gentests',
help_short='Generates test binaries.',
help_long='''
Generates test binaries (under src/xenia/cpu/frontend/testing/bin/).
Run after modifying test .s files.
''',
*args, **kwargs)
def execute(self, args, pass_args, cwd):
print('Generating test binaries...')
print('')
binutils_path = os.path.join('third_party', 'binutils-ppc-cygwin')
ppc_as = os.path.join(binutils_path, 'powerpc-none-elf-as')
ppc_ld = os.path.join(binutils_path, 'powerpc-none-elf-ld')
ppc_objdump = os.path.join(binutils_path, 'powerpc-none-elf-objdump')
ppc_nm = os.path.join(binutils_path, 'powerpc-none-elf-nm')
test_src = os.path.join('src', 'xenia', 'cpu', 'frontend', 'testing')
test_bin = os.path.join(test_src, 'bin')
# Ensure the test output path exists.
if not os.path.exists(test_bin):
os.mkdir(test_bin)
src_files = [os.path.join(root, name)
for root, dirs, files in os.walk('src')
for name in files
if name.endswith(('.s'))]
def make_unix_path(p):
"""Forces a unix path separator style, as required by binutils.
"""
return string.replace(p, os.sep, '/')
any_errors = False
for src_file in src_files:
print('- %s' % (src_file))
src_name = os.path.splitext(os.path.basename(src_file))[0]
obj_file = os.path.join(test_bin, src_name) + '.o'
shell_call([
ppc_as,
'-a32',
'-be',
'-mregnames',
'-mpower7',
'-maltivec',
'-mvsx',
'-mvmx128',
'-R',
'-o%s' % (make_unix_path(obj_file)),
make_unix_path(src_file),
])
dis_file = os.path.join(test_bin, src_name) + '.dis'
shell_call([
ppc_objdump,
'--adjust-vma=0x100000',
'-Mpower7',
'-Mvmx128',
'-D',
'-EB',
make_unix_path(obj_file),
], stdout_path=dis_file)
# Eat the first 4 lines to kill the file path that'll differ across machines.
with open(dis_file) as f:
dis_file_lines = f.readlines()
with open(dis_file, 'w') as f:
f.writelines(dis_file_lines[4:])
shell_call([
ppc_ld,
'-A powerpc:common32',
'-melf32ppc',
'-EB',
'-nostdlib',
'--oformat=binary',
'-Ttext=0x80000000',
'-e0x80000000',
'-o%s' % (make_unix_path(os.path.join(test_bin, src_name) + '.bin')),
make_unix_path(obj_file),
])
shell_call([
ppc_nm,
'--numeric-sort',
make_unix_path(obj_file),
], stdout_path=os.path.join(test_bin, src_name) + '.map')
if any_errors:
print('ERROR: failed to build one or more tests.')
return 1
return 0
class CleanCommand(Command):
"""'clean' command."""
def __init__(self, subparsers, *args, **kwargs):
super(CleanCommand, self).__init__(
subparsers,
name='clean',
help_short='Removes intermediate files and build outputs.',
*args, **kwargs)
def execute(self, args, pass_args, cwd):
print('Cleaning build artifacts...')
print('')
print('- premake clean...')
run_premake_clean()
print('')
print('Success!')
return 0
class NukeCommand(Command):
"""'nuke' command."""
def __init__(self, subparsers, *args, **kwargs):
super(NukeCommand, self).__init__(
subparsers,
name='nuke',
help_short='Removes all build/ output.',
*args, **kwargs)
def execute(self, args, pass_args, cwd):
print('Cleaning build artifacts...')
print('')
print('- removing build/...')
if os.path.isdir('build/'):
shutil.rmtree('build/')
print('')
print('- git reset to master...')
shell_call([
'git',
'reset',
'--hard',
'master',
])
print('')
print('- running premake...')
run_platform_premake()
print('')
print('Success!')
return 0
def find_xenia_source_files():
"""Gets all xenia source files in the project.
Returns:
A list of file paths.
"""
return [os.path.join(root, name)
for root, dirs, files in os.walk('src')
for name in files
if name.endswith(('.cc', '.c', '.h', '.inl'))]
def find_elemental_source_files():
"""Gets all elemental-forms source files in the project.
Returns:
A list of file paths.
"""
return [os.path.join(root, name)
for root, dirs, files in os.walk('third_party/elemental-forms/src/')
for name in files
if name.endswith(('.cc', '.c', '.h', '.inl'))]
def find_all_source_files():
"""Gets all interesting source files in the project.
Returns:
A list of file paths.
"""
return find_xenia_source_files() + find_elemental_source_files()
class LintCommand(Command):
"""'lint' command."""
def __init__(self, subparsers, *args, **kwargs):
super(LintCommand, self).__init__(
subparsers,
name='lint',
help_short='Checks for lint errors with clang-format.',
*args, **kwargs)
self.parser.add_argument(
'--all', action='store_true',
help='Lint all files, not just those changed.')
self.parser.add_argument(
'--origin', action='store_true',
help='Lints all files changed relative to origin/master.')
def execute(self, args, pass_args, cwd):
clang_format_binary = get_clang_format_binary()
difftemp = '.difftemp.txt'
if args['all']:
all_files = find_all_source_files()
print('- linting %d files' % (len(all_files)))
any_errors = False
for file_path in all_files:
if os.path.exists(difftemp): os.remove(difftemp)
ret = shell_call([
clang_format_binary,
'-output-replacements-xml',
'-style=file',
file_path,
], throw_on_error=False, stdout_path=difftemp)
with open(difftemp) as f:
had_errors = '<replacement ' in f.read()
if os.path.exists(difftemp): os.remove(difftemp)
if had_errors:
any_errors = True
print('')
print(file_path)
shell_call([
clang_format_binary,
'-style=file',
file_path,
], throw_on_error=False, stdout_path=difftemp)
shell_call([
'python',
'tools/diff.py',
file_path,
difftemp,
difftemp,
])
shell_call([
'type' if sys.platform=='win32' else 'cat',
difftemp,
])
if os.path.exists(difftemp): os.remove(difftemp)
print('')
print('')
if any_errors:
print('ERROR: 1+ diffs. Stage changes and run \'xb format\' to fix.')
return 1
else:
print('Linting completed successfully.')
return 0
else:
print('- git-clang-format --diff')
if os.path.exists(difftemp): os.remove(difftemp)
ret = shell_call([
'python',
'build_tools/third_party/clang-format/git-clang-format',
'--binary=%s' % (clang_format_binary),
'--commit=%s' % ('origin/master' if args['origin'] else 'HEAD'),
'--diff',
], throw_on_error=False, stdout_path=difftemp)
with open(difftemp) as f:
contents = f.read()
not_modified = 'no modified files' in contents
not_modified = not_modified or 'did not modify' in contents
f.close()
if os.path.exists(difftemp): os.remove(difftemp)
if not not_modified:
any_errors = True
print('')
shell_call([
'python',
'build_tools/third_party/clang-format/git-clang-format',
'--binary=%s' % (clang_format_binary),
'--commit=%s' % ('origin/master' if args['origin'] else 'HEAD'),
'--diff',
])
print('ERROR: 1+ diffs. Stage changes and run \'xb format\' to fix.')
return 1
else:
print('Linting completed successfully.')
return 0
class FormatCommand(Command):
"""'format' command."""
def __init__(self, subparsers, *args, **kwargs):
super(FormatCommand, self).__init__(
subparsers,
name='format',
help_short='Reformats staged code with clang-format.',
*args, **kwargs)
self.parser.add_argument(
'--all', action='store_true',
help='Format all files, not just those changed.')
self.parser.add_argument(
'--origin', action='store_true',
help='Formats all files changed relative to origin/master.')
def execute(self, args, pass_args, cwd):
clang_format_binary = get_clang_format_binary()
if args['all']:
all_files = find_all_source_files()
print('- clang-format [%d files]' % (len(all_files)))
any_errors = False
for file_path in all_files:
ret = shell_call([
clang_format_binary,
'-i',
'-style=file',
file_path,
], throw_on_error=False)
if ret:
any_errors = True
print('')
if any_errors:
print('ERROR: 1+ clang-format calls failed.')
print('Ensure all files are staged.')
return 1
else:
print('Formatting completed successfully.')
return 0
else:
print('- git-clang-format')
shell_call([
'python',
'build_tools/third_party/clang-format/git-clang-format',
'--binary=%s' % (clang_format_binary),
'--commit=%s' % ('origin/master' if args['origin'] else 'HEAD'),
])
print('')
return 0
# TODO(benvanik): merge into linter, or as lint --anal?
class StyleCommand(Command):
"""'style' command."""
def __init__(self, subparsers, *args, **kwargs):
super(StyleCommand, self).__init__(
subparsers,
name='style',
help_short='Runs the style checker on all code.',
*args, **kwargs)
def execute(self, args, pass_args, cwd):
all_files = [file_path for file_path in find_all_source_files()
if not file_path.endswith('_test.cc')]
print('- cpplint [%d files]' % (len(all_files)))
ret = shell_call([
'python',
'build_tools/third_party/google-styleguide/cpplint/cpplint.py',
'--output=vs7',
'--linelength=80',
'--filter=-build/c++11',
'--root=src',
] + all_files, throw_on_error=False)
print('')
if ret:
print('ERROR: 1+ cpplint calls failed.')
return 1
else:
print('Style linting completed successfully.')
return 0
return 0
class DevenvCommand(Command):
"""'devenv' command."""
def __init__(self, subparsers, *args, **kwargs):
super(DevenvCommand, self).__init__(
subparsers,
name='devenv',
help_short='Launches Visual Studio with the sln.',
*args, **kwargs)
def execute(self, args, pass_args, cwd):
print('Launching Visual Studio...')
print('')
print('- running premake...')
run_platform_premake()
print('')
print('- launching devenv...')
shell_call([
'devenv',
'build\\xenia.sln',
])
print('')
return 0
if __name__ == '__main__':
main()