From ff9ecfa942977c72e17744bac0b0e69c93c033b9 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Wed, 15 Mar 2017 14:54:10 -0700 Subject: [PATCH] Bug 1314147 - Add 'mach vendor aom' for maintaining av1 codec support. r=froydnj We've traditionally had per-directory 'update' scripts for third-party media codec implementations. Instead, leverage the new 'mach vendor' command to centralize the import and build file generation, placing the upstream source in third_party/aom. Note this includes another copy of gtest and other dependencies which we don't use, but they're required by the upstream build process we use to generate our own build description, so I've left them in for now. MozReview-Commit-ID: CnWcSwvQZEh --HG-- extra : rebase_source : 28172a41332e920c9ea4a475a6990d43ebf8185f --- python/mozbuild/mozbuild/mach_commands.py | 13 ++ python/mozbuild/mozbuild/vendor_aom.py | 157 ++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 python/mozbuild/mozbuild/vendor_aom.py diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 2dae79ff442a..ba6c5aa6b38f 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -1836,6 +1836,19 @@ class Vendor(MachCommandBase): vendor_command = self._spawn(VendorRust) vendor_command.vendor(**kwargs) + @SubCommand('vendor', 'aom', + description='Vendor av1 video codec reference implementation into the source repository.') + @CommandArgument('-r', '--revision', + help='Repository tag or commit to update to.') + @CommandArgument('--ignore-modified', action='store_true', + help='Ignore modified files in current checkout', + default=False) + def vendor_aom(self, **kwargs): + from mozbuild.vendor_aom import VendorAOM + vendor_command = self._spawn(VendorAOM) + vendor_command.vendor(**kwargs) + + @CommandProvider class WebRTCGTestCommands(GTestCommands): @Command('webrtc-gtest', category='testing', diff --git a/python/mozbuild/mozbuild/vendor_aom.py b/python/mozbuild/mozbuild/vendor_aom.py new file mode 100644 index 000000000000..71c710ebc8dc --- /dev/null +++ b/python/mozbuild/mozbuild/vendor_aom.py @@ -0,0 +1,157 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from distutils.version import LooseVersion +import logging +from mozbuild.base import ( + BuildEnvironmentNotFoundException, + MozbuildObject, +) +import mozfile +import mozpack.path as mozpath +import requests +import re +import sys +import tarfile + +class VendorAOM(MozbuildObject): + base_url = 'https://aomedia.googlesource.com/aom/' + + def upstream_url(self, revision): + '''Construct a url for a tarball snapshot of the given revision.''' + return mozpath.join(self.base_url, '+archive', revision + '.tar.gz') + + def upstream_commit(self, revision): + '''Convert a revision to a git commit and timestamp. + + Ask the upstream repo to convert the requested revision to + a git commit id and timestamp, so we can be precise in + what we're vendoring.''' + url = mozpath.join(self.base_url, '+', revision + '?format=JSON') + self.log(logging.INFO, 'fetch', {'url': url}, + 'Fetching commit id from {url}') + req = requests.get(url) + req.raise_for_status() + try: + info = req.json() + except ValueError as e: + if 'No JSON object' in e.message: + # As of 2017 May, googlesource sends 4 garbage characters + # at the beginning of the json response. Work around this. + # https://bugs.chromium.org/p/chromium/issues/detail?id=718550 + import json + info = json.loads(req.text[4:]) + else: + raise + return (info['commit'], info['committer']['time']) + + def fetch_and_unpack(self, revision, target): + '''Fetch and unpack upstream source''' + url = self.upstream_url(revision) + self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}') + filename = 'libaom-' + revision + '.tar.gz' + with open(filename, 'wb') as f: + req = requests.get(url, stream=True) + for data in req.iter_content(4096): + f.write(data) + tar = tarfile.open(filename) + bad_paths = filter(lambda name: name.startswith('/') or '..' in name, + tar.getnames()) + if any(bad_paths): + raise Exception("Tar archive contains non-local paths," + "e.g. '%s'" % bad_paths[0]) + self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target) + mozfile.remove(target) + self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.') + tar.extractall(target) + mozfile.remove(filename) + + def update_readme(self, revision, timestamp, target): + filename = mozpath.join(target, 'README_MOZILLA') + with open(filename) as f: + readme = f.read() + + prefix = 'The git commit ID used was' + if prefix in readme: + new_readme = re.sub(prefix + ' [v\.a-f0-9]+.*$', + prefix + ' %s (%s).' % (revision, timestamp), + readme) + else: + new_readme = '%s\n\n%s %s.' % (readme, prefix, revision) + + if readme != new_readme: + with open(filename, 'w') as f: + f.write(new_readme) + + def clean_upstream(self, target): + '''Remove files we don't want to import.''' + mozfile.remove(mozpath.join(target, '.gitattributes')) + mozfile.remove(mozpath.join(target, '.gitignore')) + mozfile.remove(mozpath.join(target, 'build', '.gitattributes')) + mozfile.remove(mozpath.join(target, 'build' ,'.gitignore')) + + def generate_sources(self, target): + ''' + Run the library's native build system to update ours. + + Invoke configure for each supported platform to generate + appropriate config and header files, then invoke the + makefile to obtain a list of source files, writing + these out in the appropriate format for our build + system to use. + ''' + config_dir = mozpath.join(target, 'config') + self.log(logging.INFO, 'rm_confg_dir', {}, 'rm -rf %s' % config_dir) + mozfile.remove(config_dir) + self.run_process(args=['./generate_sources_mozbuild.sh'], + cwd=target) + + def check_modified_files(self): + ''' + Ensure that there aren't any uncommitted changes to files + in the working copy, since we're going to change some state + on the user. + ''' + modified = self.repository.get_modified_files() + if modified: + self.log(logging.ERROR, 'modified_files', {}, + '''You have uncommitted changes to the following files: + +{files} + +Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`. +'''.format(files='\n'.join(sorted(modified)))) + sys.exit(1) + + def vendor(self, revision, ignore_modified=False): + self.populate_logger() + self.log_manager.enable_unstructured() + + if not ignore_modified: + self.check_modified_files() + if not revision: + revision = 'master' + commit, timestamp = self.upstream_commit(revision) + + vendor_dir = mozpath.join(self.topsrcdir, 'third_party/aom') + self.fetch_and_unpack(commit, vendor_dir) + self.log(logging.INFO, 'clean_upstream', {}, + '''Removing unnecessary files.''') + self.clean_upstream(vendor_dir) + glue_dir = mozpath.join(self.topsrcdir, 'media/libaom') + self.log(logging.INFO, 'generate_sources', {}, + '''Generating build files...''') + self.generate_sources(glue_dir) + self.log(logging.INFO, 'update_readme', {}, + '''Updating README_MOZILLA.''') + self.update_readme(commit, timestamp, glue_dir) + self.repository.add_remove_files(vendor_dir) + self.log(logging.INFO, 'add_remove_files', {}, + '''Registering changes with version control.''') + self.repository.add_remove_files(vendor_dir) + self.repository.add_remove_files(glue_dir) + self.log(logging.INFO, 'done', {'revision': revision}, + '''Update to aom version '{revision}' ready to commit.''')