From 04992e38a19508f3aa766c05f5d0f240abda5138 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Mon, 22 May 2017 18:11:19 -0500 Subject: [PATCH] servo: Merge #16565 - Convert nightly upload script to Python (from aneeshusa:move-upload-nightly-script-to-python); r=metajack Now that MinGW and MSYS have been removed from the Windows builders, bash is not available to run the previous upload_nightlies.sh script. Convert the script to Python 2 for cross-platform support. Additionally, switch to the `boto3` library for easy uploading without needing to install `s3cmd`, and move the code into mach for easy `boto3` installation as the new `./mach upload-nightly` command. Also, hard-code the paths to the packages instead of using globs to look for them, as the paths are static. (The paths used to contain timestamps, but we now insert timestamps when uploading to S3 to improve reproducibility.) --- - [ ] `./mach build -d` does not report any errors - [x] `./mach test-tidy` does not report any errors - [x] These changes fix #16560 (github issue number if applicable). - [ ] There are tests for these changes OR - [x] These changes do not require tests because Buildbot will test them. Source-Repo: https://github.com/servo/servo Source-Revision: 9468fae0d443b2a8b9a7694f917dc097ac6c7e60 --HG-- extra : subtree_source : https%3A//hg.mozilla.org/projects/converted-servo-linear extra : subtree_revision : 593f32ad93c14cb66dd36dfdb41bb36b41a0c558 --- servo/etc/ci/buildbot_steps.yml | 10 +- servo/etc/ci/upload_nightly.sh | 113 ----------------------- servo/python/requirements.txt | 3 + servo/python/servo/package_commands.py | 122 ++++++++++++++++++++++++- 4 files changed, 129 insertions(+), 119 deletions(-) delete mode 100755 servo/etc/ci/upload_nightly.sh diff --git a/servo/etc/ci/buildbot_steps.yml b/servo/etc/ci/buildbot_steps.yml index 529cf56caee5..b3527e0da01d 100644 --- a/servo/etc/ci/buildbot_steps.yml +++ b/servo/etc/ci/buildbot_steps.yml @@ -38,8 +38,8 @@ mac-nightly: - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./mach package --release - - ./etc/ci/upload_nightly.sh mac - - ./etc/ci/upload_nightly.sh macbrew + - ./mach upload-nightly mac + - ./mach upload-nightly macbrew linux-rel-intermittent: - ./mach clean-nightlies --keep 3 --force @@ -89,7 +89,7 @@ linux-nightly: - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./mach package --release - - ./etc/ci/upload_nightly.sh linux + - ./mach upload-nightly linux android: - ./mach clean-nightlies --keep 3 --force @@ -103,7 +103,7 @@ android-nightly: - ./mach clean-nightlies --keep 3 --force - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ANDROID_SDK=/home/servo/android/sdk/r25.2.3 ./mach build --android --release - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ANDROID_SDK=/home/servo/android/sdk/r25.2.3 ./mach package --android --release - - ./etc/ci/upload_nightly.sh android + - ./mach upload-nightly android arm32: - ./mach clean-nightlies --keep 3 --force @@ -128,4 +128,4 @@ windows-msvc-nightly: - mach.bat clean-nightlies --keep 3 --force - mach.bat build --release - mach.bat package --release - - bash -l ./etc/ci/upload_nightly.sh windows-msvc + - mach.bat upload-nightly windows-msvc diff --git a/servo/etc/ci/upload_nightly.sh b/servo/etc/ci/upload_nightly.sh deleted file mode 100755 index fbdb7ba99ad5..000000000000 --- a/servo/etc/ci/upload_nightly.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -set -o errexit -set -o nounset -set -o pipefail -shopt -s failglob - - -usage() { - printf \ - "usage: %s android|linux|mac|macbrew|windows-msvc\n" \ - "${0}" -} - - -upload() { - local nightly_filename - nightly_filename="${4}-$(basename "${2}")" - local -r nightly_upload_dir="s3://servo-builds/nightly/${1}" - local -r package_upload_path="${nightly_upload_dir}/${nightly_filename}" - s3cmd --mime-type="application/octet-stream" \ - put "${2}" "${package_upload_path}" - s3cmd cp "${package_upload_path}" "${nightly_upload_dir}/servo-latest.${3}" -} - -update_brew() { - echo "Updating brew formula" - - local package_url sha version script_dir tmp_dir nightly_filename - - nightly_filename="${2}-$(basename "${1}")" - package_url="https://download.servo.org/nightly/macbrew/${nightly_filename}" - sha="$(shasum -a 256 "${1}" | sed -e 's/ .*//')" - - # This will transform a timestamp (2016-12-13T08-01-10Z for example) - # into a valid brew version number (2016.12.13). - version="$(echo "${2}" | \ - sed -n 's/\([0-9]\{4\}\)-\([0-9]\{2\}\)-\([0-9]\{2\}\).*/\1.\2.\3/p')" - - script_dir="${PWD}/$(dirname "${0}")" - tmp_dir="$(mktemp -d -t homebrew-servo.XXXXX)" - - git -C "${tmp_dir}" clone https://github.com/servo/homebrew-servo.git . - - # Not using "/" as it's used in PACKAGEURL - sed "s|PACKAGEURL|${package_url}|g - s|SHA|${sha}|g - s|VERSION|${version}|g" \ - < "${script_dir}/servo-binary-formula.rb.in" \ - > "${tmp_dir}/Formula/servo-bin.rb" - - git -C "${tmp_dir}" add ./Formula/servo-bin.rb - git -C "${tmp_dir}" \ - -c user.name="Tom Servo" \ - -c user.email="servo@servo.org" \ - commit \ - --message="Version bump: ${version}" - - git -C "${tmp_dir}" push -qf \ - "https://${GITHUB_HOMEBREW_TOKEN}@github.com/servo/homebrew-servo.git" \ - master >/dev/null 2>&1 - - rm -rf "${tmp_dir}" -} - -main() { - if (( "${#}" != 1 )); then - usage >&2 - return 1 - fi - - local platform package extension nightly_timestamp - platform="${1}" - nightly_timestamp="$(date -u +"%Y-%m-%dT%H-%M-%SZ")" - - if [[ "${platform}" == "android" ]]; then - extension=apk - package=target/arm-linux-androideabi/release/*."${extension}" - elif [[ "${platform}" == "linux" ]]; then - extension=tar.gz - package=target/release/*."${extension}" - elif [[ "${platform}" == "mac" ]]; then - extension=dmg - package=target/release/*."${extension}" - elif [[ "${platform}" == "macbrew" ]]; then - extension=tar.gz - package=target/release/brew/*."${extension}" - elif [[ "${platform}" == "windows-msvc" ]]; then - extension=msi - package=target/release/msi/*.msi - upload "${platform}" ${package} "${extension}" "${nightly_timestamp}" - extension=zip - package=target/release/msi/*.zip - else - usage >&2 - return 1 - fi - - # Lack of quotes on package allows glob expansion - # Note that this is not robust in the case of embedded spaces - # TODO(aneeshusa): make this glob robust using e.g. arrays or Python - upload "${platform}" ${package} "${extension}" "${nightly_timestamp}" - - if [[ "${platform}" == "macbrew" ]]; then - update_brew ${package} "${nightly_timestamp}" - fi -} - -main "${@}" diff --git a/servo/python/requirements.txt b/servo/python/requirements.txt index 41b25ccdb5a1..b96c4741b051 100644 --- a/servo/python/requirements.txt +++ b/servo/python/requirements.txt @@ -24,4 +24,7 @@ ply == 3.8 # For Cross-platform colored terminal text colorama == 0.3.7 +# For package uploading +boto3 == 1.4.4 + -e python/tidy diff --git a/servo/python/servo/package_commands.py b/servo/python/servo/package_commands.py index ad842c9258a4..57294f00763b 100644 --- a/servo/python/servo/package_commands.py +++ b/servo/python/servo/package_commands.py @@ -7,13 +7,17 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +from datetime import datetime +import hashlib import json import os import os.path as path import shutil import subprocess +import sys +import tempfile from mach.decorators import ( CommandArgument, @@ -35,6 +39,26 @@ from servo.command_base import ( from servo.util import delete +PACKAGES = { + 'android': [ + 'target/arm-linux-androideabi/release/servo.apk', + ], + 'linux': [ + 'target/release/servo-tech-demo.tar.gz', + ], + 'mac': [ + 'target/release/servo-tech-demo.dmg', + ], + 'macbrew': [ + 'target/release/brew/servo.tar.gz', + ], + 'windows-msvc': [ + r'target\release\msi\Servo.msi', + r'target\release\msi\Servo.zip', + ], +} + + def otool(s): o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE) for l in o.stdout: @@ -374,3 +398,99 @@ class PackageCommands(CommandBase): print(" ".join(exec_command)) return subprocess.call(exec_command, env=self.build_env()) + + @Command('upload-nightly', + description='Upload Servo nightly to S3', + category='package') + @CommandArgument('platform', + choices=PACKAGES.keys(), + help='Package platform type to upload') + def upload_nightly(self, platform): + import boto3 + + def nightly_filename(package, timestamp): + return '{}-{}'.format( + timestamp.isoformat() + 'Z', # The `Z` denotes UTC + path.basename(package) + ) + + def upload_to_s3(platform, package, timestamp): + s3 = boto3.client('s3') + BUCKET = 'servo-builds' + + nightly_dir = 'nightly/{}'.format(platform) + filename = nightly_filename(package, timestamp) + package_upload_key = '{}/{}'.format(nightly_dir, filename) + extension = path.basename(package).partition('.')[2] + latest_upload_key = '{}/servo-latest.{}'.format(nightly_dir, extension) + + s3.upload_file(package, BUCKET, package_upload_key) + copy_source = { + 'Bucket': BUCKET, + 'Key': package_upload_key, + } + s3.copy(copy_source, BUCKET, latest_upload_key) + + def update_brew(package, timestamp): + print("Updating brew formula") + + package_url = 'https://download.servo.org/nightly/macbrew/{}'.format( + nightly_filename(package, timestamp) + ) + with open(package) as p: + digest = hashlib.sha256(p.read()).hexdigest() + + brew_version = timestamp.strftime('%Y.%m.%d') + + with tempfile.TemporaryDirectory(prefix='homebrew-servo') as tmp_dir: + def call_git(cmd, **kwargs): + subprocess.check_call( + ['git', '-C', tmp_dir] + cmd, + **kwargs + ) + + call_git([ + 'clone', + 'https://github.com/servo/homebrew-servo.git', + '.', + ]) + + script_dir = path.dirname(path.realpath(__file__)) + with open(path.join(script_dir, 'servo-binary-formula.rb.in')) as f: + formula = f.read() + formula = formula.replace('PACKAGEURL', package_url) + formula = formula.replace('SHA', digest) + formula = formula.replace('VERSION', brew_version) + with open(path.join(tmp_dir, 'Formula', 'servo-bin.rb')) as f: + f.write(formula) + + call_git(['add', path.join('.', 'Formula', 'servo-bin.rb')]) + call_git([ + '-c', 'user.name=Tom Servo', + '-c', 'user.email=servo@servo.org', + 'commit', + '--message=Version Bump: {}'.format(brew_version), + ]) + + token = os.environ['GITHUB_HOMEBREW_TOKEN'] + call_git([ + 'push', + '-qf', + 'https://{}@github.com/servo/homebrew-servo.git'.format(token), + 'master', + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + timestamp = datetime.utcnow().replace(microsecond=0) + for package in PACKAGES[platform]: + if not path.isfile(package): + print("Could not find package for {} at {}".format( + platform, + package + ), file=sys.stderr) + return 1 + upload_to_s3(platform, package, timestamp) + + if platform == 'macbrew': + update_brew(package, timestamp) + + return 0