mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-24 21:31:04 +00:00
Bug 1425137: Remove usused rust-build docker image; r=dustin
MozReview-Commit-ID: 7fQ8JtZIalM --HG-- extra : rebase_source : c1b1c9d70483943289f030cc35cbe8650112f8fa
This commit is contained in:
Tom Prince
parent
261ab4c0c7
commit
66fabdfc27
@ -1,32 +0,0 @@
|
||||
FROM quay.io/rust/rust-buildbot
|
||||
MAINTAINER Ralph Giles <giles@mozilla.com>
|
||||
|
||||
# Reset user/workdir from parent image so we can install software.
|
||||
WORKDIR /
|
||||
USER root
|
||||
|
||||
# Install tooltool directly from github.
|
||||
ADD https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py /build/tooltool.py
|
||||
RUN chmod +rx /build/tooltool.py
|
||||
|
||||
# Add build scripts.
|
||||
ADD fetch_rust.sh build_rust.sh /build/
|
||||
ADD fetch_cargo.sh build_cargo.sh /build/
|
||||
ADD package_rust.sh upload_rust.sh /build/
|
||||
ADD repack_rust.py splat_rust.py /build/
|
||||
RUN chmod +x /build/*
|
||||
|
||||
# Create user for doing the build.
|
||||
ENV USER worker
|
||||
ENV HOME /home/${USER}
|
||||
|
||||
RUN useradd -d ${HOME} -m ${USER}
|
||||
|
||||
# Set up the user's tree
|
||||
WORKDIR ${HOME}
|
||||
|
||||
# Invoke our build scripts by default, but allow other commands.
|
||||
USER ${USER}
|
||||
ENTRYPOINT /build/fetch_rust.sh && /build/build_rust.sh && \
|
||||
/build/fetch_cargo.sh && /build/build_cargo.sh && \
|
||||
/build/package_rust.sh && /build/upload_rust.sh
|
||||
@ -1,2 +0,0 @@
|
||||
This is a docker script for building rust toolchains for
|
||||
use in Mozilla's build clusters.
|
||||
@ -1 +0,0 @@
|
||||
quay.io/rust
|
||||
@ -1 +0,0 @@
|
||||
0.4.8
|
||||
@ -1,30 +0,0 @@
|
||||
#!/bin/bash -vex
|
||||
|
||||
set -x -e
|
||||
|
||||
: WORKSPACE ${WORKSPACE:=/home/worker}
|
||||
: BRANCH ${BRANCH:=0.15.0}
|
||||
|
||||
set -v
|
||||
|
||||
# Configure and build cargo.
|
||||
|
||||
if test $(uname -s) = "Darwin"; then
|
||||
export MACOSX_DEPLOYMENT_TARGET=10.7
|
||||
fi
|
||||
|
||||
# Build the initial cargo checkout, which can download a snapshot.
|
||||
pushd ${WORKSPACE}/cargo
|
||||
./configure --prefix=${WORKSPACE}/rustc --local-rust-root=${WORKSPACE}/rustc
|
||||
make
|
||||
make dist
|
||||
make install
|
||||
popd
|
||||
|
||||
# Build the version we want.
|
||||
export PATH=$PATH:${WORKSPACE}/rustc/bin
|
||||
pushd ${WORKSPACE}/cargo
|
||||
make clean
|
||||
git checkout ${BRANCH}
|
||||
OPENSSL_DIR=/rustroot/cargo64 cargo install --root=${WORKSPACE}/rustc --force
|
||||
popd
|
||||
@ -1,28 +0,0 @@
|
||||
#!/bin/bash -vex
|
||||
|
||||
set -x -e
|
||||
|
||||
: WORKSPACE ${WORKSPACE:=/home/worker}
|
||||
|
||||
CORES=$(nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu)
|
||||
|
||||
set -v
|
||||
|
||||
# Configure and build rust.
|
||||
OPTIONS="--enable-llvm-static-stdcpp --disable-docs"
|
||||
OPTIONS+="--enable-debuginfo"
|
||||
OPTIONS+="--release-channel=stable"
|
||||
i586="i586-unknown-linux-gnu"
|
||||
i686="i686-unknown-linux-gnu"
|
||||
x64="x86_64-unknown-linux-gnu"
|
||||
arm_android="arm-linux-androideabi"
|
||||
x86_android="i686-linux-android"
|
||||
|
||||
mkdir -p ${WORKSPACE}/rust-build
|
||||
pushd ${WORKSPACE}/rust-build
|
||||
${WORKSPACE}/rust/configure --prefix=${WORKSPACE}/rustc \
|
||||
--target=${x64},${i686} ${OPTIONS}
|
||||
make -j ${CORES}
|
||||
make dist
|
||||
make install
|
||||
popd
|
||||
@ -1,36 +0,0 @@
|
||||
#!/bin/bash -vex
|
||||
|
||||
set -e
|
||||
|
||||
: WORKSPACE ${WORKSPACE:=$PWD}
|
||||
: TOOLTOOL ${TOOLTOOL:=python $WORKSPACE/tooltool.py}
|
||||
|
||||
CORES=$(nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu)
|
||||
echo Building on $CORES cpus...
|
||||
|
||||
OPTIONS="--enable-debuginfo --disable-docs"
|
||||
TARGETS="x86_64-apple-darwin,i686-apple-darwin"
|
||||
|
||||
PREFIX=${WORKSPACE}/rustc
|
||||
|
||||
set -v
|
||||
|
||||
mkdir -p ${WORKSPACE}/gecko-rust-mac
|
||||
pushd ${WORKSPACE}/gecko-rust-mac
|
||||
|
||||
export MACOSX_DEPLOYMENT_TARGET=10.7
|
||||
${WORKSPACE}/rust/configure --prefix=${PREFIX} --target=${TARGETS} ${OPTIONS}
|
||||
make -j ${CORES}
|
||||
|
||||
rm -rf ${PREFIX}
|
||||
mkdir ${PREFIX}
|
||||
make dist
|
||||
make install
|
||||
popd
|
||||
|
||||
# Package the toolchain for upload.
|
||||
pushd ${WORKSPACE}
|
||||
rustc/bin/rustc --version
|
||||
tar cvjf rustc.tar.bz2 rustc/*
|
||||
${TOOLTOOL} add --visibility=public --unpack rustc.tar.bz2
|
||||
popd
|
||||
@ -1,21 +0,0 @@
|
||||
#!/bin/bash -vex
|
||||
|
||||
set -x -e
|
||||
|
||||
# Inputs, with defaults
|
||||
|
||||
: REPOSITORY ${REPOSITORY:=https://github.com/rust-lang/cargo}
|
||||
: BRANCH ${BRANCH:=0.14.0}
|
||||
|
||||
: WORKSPACE ${WORKSPACE:=/home/worker}
|
||||
|
||||
set -v
|
||||
|
||||
# Check out rust sources
|
||||
SRCDIR=${WORKSPACE}/cargo
|
||||
git clone --recursive $REPOSITORY -b $BRANCH ${SRCDIR}
|
||||
|
||||
# Report version
|
||||
VERSION=$(git -C ${SRCDIR} describe --tags --dirty)
|
||||
COMMIT=$(git -C ${SRCDIR} rev-parse HEAD)
|
||||
echo "cargo ${VERSION} (commit ${COMMIT})" | tee cargo-version
|
||||
@ -1,20 +0,0 @@
|
||||
#!/bin/bash -vex
|
||||
|
||||
set -x -e
|
||||
|
||||
# Inputs, with defaults
|
||||
|
||||
: RUST_REPOSITORY ${RUST_REPOSITORY:=https://github.com/rust-lang/rust}
|
||||
: RUST_BRANCH ${RUST_BRANCH:=stable}
|
||||
|
||||
: WORKSPACE ${WORKSPACE:=/home/worker}
|
||||
|
||||
set -v
|
||||
|
||||
# Check out rust sources
|
||||
git clone $RUST_REPOSITORY -b $RUST_BRANCH ${WORKSPACE}/rust
|
||||
|
||||
# Report version
|
||||
VERSION=$(git -C ${WORKSPACE}/rust describe --tags --dirty)
|
||||
COMMIT=$(git -C ${WORKSPACE}/rust rev-parse HEAD)
|
||||
echo "rust ${VERSION} (commit ${COMMIT})" | tee rust-version
|
||||
@ -1,13 +0,0 @@
|
||||
#!/bin/bash -vex
|
||||
|
||||
set -x -e
|
||||
|
||||
: WORKSPACE ${WORKSPACE:=/home/worker}
|
||||
|
||||
set -v
|
||||
|
||||
# Package the toolchain for upload.
|
||||
pushd ${WORKSPACE}
|
||||
tar cvJf rustc.tar.xz rustc/*
|
||||
/build/tooltool.py add --visibility=public --unpack rustc.tar.xz
|
||||
popd
|
||||
@ -1,240 +0,0 @@
|
||||
#!/bin/env python
|
||||
'''
|
||||
This script downloads and repacks official rust language builds
|
||||
with the necessary tool and target support for the Firefox
|
||||
build environment.
|
||||
'''
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import argparse
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
import requests
|
||||
import subprocess
|
||||
import toml
|
||||
|
||||
|
||||
def log(msg):
|
||||
print('repack: %s' % msg)
|
||||
|
||||
|
||||
def fetch_file(url):
|
||||
'''Download a file from the given url if it's not already present.'''
|
||||
filename = os.path.basename(url)
|
||||
if os.path.exists(filename):
|
||||
return
|
||||
r = requests.get(url, stream=True)
|
||||
r.raise_for_status()
|
||||
with open(filename, 'wb') as fd:
|
||||
for chunk in r.iter_content(4096):
|
||||
fd.write(chunk)
|
||||
|
||||
|
||||
def sha256sum():
|
||||
'''Return the command for verifying SHA-2 256-bit checksums.'''
|
||||
if sys.platform.startswith('darwin'):
|
||||
return 'shasum'
|
||||
else:
|
||||
return 'sha256sum'
|
||||
|
||||
|
||||
def fetch(url):
|
||||
'''Download and verify a package url.'''
|
||||
base = os.path.basename(url)
|
||||
log('Fetching %s...' % base)
|
||||
fetch_file(url + '.asc')
|
||||
fetch_file(url)
|
||||
fetch_file(url + '.sha256')
|
||||
log('Verifying %s...' % base)
|
||||
shasum = sha256sum()
|
||||
subprocess.check_call([shasum, '-c', base + '.sha256'])
|
||||
subprocess.check_call(['gpg', '--verify', base + '.asc', base])
|
||||
if True:
|
||||
subprocess.check_call([
|
||||
'keybase', 'pgp', 'verify', '-d', base + '.asc', '-i', base,
|
||||
])
|
||||
|
||||
|
||||
def install(filename, target):
|
||||
'''Run a package's installer script against the given target directory.'''
|
||||
log('Unpacking %s...' % filename)
|
||||
subprocess.check_call(['tar', 'xf', filename])
|
||||
basename = filename.split('.tar')[0]
|
||||
log('Installing %s...' % basename)
|
||||
install_cmd = [os.path.join(basename, 'install.sh')]
|
||||
install_cmd += ['--prefix=' + os.path.abspath(target)]
|
||||
install_cmd += ['--disable-ldconfig']
|
||||
subprocess.check_call(install_cmd)
|
||||
log('Cleaning %s...' % basename)
|
||||
subprocess.check_call(['rm', '-rf', basename])
|
||||
|
||||
|
||||
def package(manifest, pkg, target):
|
||||
'''Pull out the package dict for a particular package and target
|
||||
from the given manifest.'''
|
||||
version = manifest['pkg'][pkg]['version']
|
||||
info = manifest['pkg'][pkg]['target'][target]
|
||||
return (version, info)
|
||||
|
||||
|
||||
def fetch_package(manifest, pkg, host):
|
||||
version, info = package(manifest, pkg, host)
|
||||
log('%s %s\n %s\n %s' % (pkg, version, info['url'], info['hash']))
|
||||
if not info['available']:
|
||||
log('%s marked unavailable for %s' % (pkg, host))
|
||||
raise AssertionError
|
||||
fetch(info['url'])
|
||||
return info
|
||||
|
||||
|
||||
def fetch_std(manifest, targets):
|
||||
stds = []
|
||||
for target in targets:
|
||||
info = fetch_package(manifest, 'rust-std', target)
|
||||
stds.append(info)
|
||||
return stds
|
||||
|
||||
|
||||
def tar_for_host(host):
|
||||
if 'linux' in host:
|
||||
tar_options = 'cJf'
|
||||
tar_ext = '.tar.xz'
|
||||
else:
|
||||
tar_options = 'cjf'
|
||||
tar_ext = '.tar.bz2'
|
||||
return tar_options, tar_ext
|
||||
|
||||
|
||||
def fetch_manifest(channel='stable'):
|
||||
url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
|
||||
req = requests.get(url)
|
||||
req.raise_for_status()
|
||||
manifest = toml.loads(req.content)
|
||||
if manifest['manifest-version'] != '2':
|
||||
raise NotImplementedError('Unrecognized manifest version %s.' %
|
||||
manifest['manifest-version'])
|
||||
return manifest
|
||||
|
||||
|
||||
def repack(host, targets, channel='stable', suffix='', cargo_channel=None):
|
||||
log("Repacking rust for %s..." % host)
|
||||
|
||||
manifest = fetch_manifest(channel)
|
||||
log('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
|
||||
if cargo_channel == channel:
|
||||
cargo_manifest = manifest
|
||||
else:
|
||||
cargo_manifest = fetch_manifest(cargo_channel)
|
||||
log('Using manifest for cargo %s as of %s.' %
|
||||
(cargo_channel, cargo_manifest['date']))
|
||||
|
||||
log('Fetching packages...')
|
||||
rustc = fetch_package(manifest, 'rustc', host)
|
||||
cargo = fetch_package(cargo_manifest, 'cargo', host)
|
||||
stds = fetch_std(manifest, targets)
|
||||
|
||||
log('Installing packages...')
|
||||
tar_basename = 'rustc-' + host
|
||||
if suffix:
|
||||
tar_basename += '-' + suffix
|
||||
tar_basename += '-repack'
|
||||
install_dir = 'rustc'
|
||||
subprocess.check_call(['rm', '-rf', install_dir])
|
||||
install(os.path.basename(rustc['url']), install_dir)
|
||||
install(os.path.basename(cargo['url']), install_dir)
|
||||
for std in stds:
|
||||
install(os.path.basename(std['url']), install_dir)
|
||||
pass
|
||||
|
||||
log('Tarring %s...' % tar_basename)
|
||||
tar_options, tar_ext = tar_for_host(host)
|
||||
subprocess.check_call(
|
||||
['tar', tar_options, tar_basename + tar_ext, install_dir])
|
||||
subprocess.check_call(['rm', '-rf', install_dir])
|
||||
|
||||
|
||||
def repack_cargo(host, channel='nightly'):
|
||||
log('Repacking cargo for %s...' % host)
|
||||
# Cargo doesn't seem to have a .toml manifest.
|
||||
base_url = 'https://static.rust-lang.org/cargo-dist/'
|
||||
req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel))
|
||||
req.raise_for_status()
|
||||
file = ''
|
||||
for line in req.iter_lines():
|
||||
if line.find(host) != -1:
|
||||
file = line.strip()
|
||||
if not file:
|
||||
log('No manifest entry for %s!' % host)
|
||||
return
|
||||
manifest = {
|
||||
'date': req.headers['Last-Modified'],
|
||||
'pkg': {
|
||||
'cargo': {
|
||||
'version': channel,
|
||||
'target': {
|
||||
host: {
|
||||
'url': os.path.join(base_url, file),
|
||||
'hash': None,
|
||||
'available': True,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
log('Using manifest for cargo %s.' % channel)
|
||||
log('Fetching packages...')
|
||||
cargo = fetch_package(manifest, 'cargo', host)
|
||||
log('Installing packages...')
|
||||
install_dir = 'cargo'
|
||||
subprocess.check_call(['rm', '-rf', install_dir])
|
||||
install(os.path.basename(cargo['url']), install_dir)
|
||||
tar_basename = 'cargo-%s-repack' % host
|
||||
log('Tarring %s...' % tar_basename)
|
||||
tar_options, tar_ext = tar_for_host(host)
|
||||
subprocess.check_call(
|
||||
['tar', tar_options, tar_basename + tar_ext, install_dir])
|
||||
subprocess.check_call(['rm', '-rf', install_dir])
|
||||
|
||||
|
||||
# rust platform triples
|
||||
android = "armv7-linux-androideabi"
|
||||
android_x86 = "i686-linux-android"
|
||||
android_aarch64 = "aarch64-linux-android"
|
||||
linux64 = "x86_64-unknown-linux-gnu"
|
||||
linux32 = "i686-unknown-linux-gnu"
|
||||
mac64 = "x86_64-apple-darwin"
|
||||
mac32 = "i686-apple-darwin"
|
||||
win64 = "x86_64-pc-windows-msvc"
|
||||
win32 = "i686-pc-windows-msvc"
|
||||
mingw32 = "i686-pc-windows-gnu"
|
||||
|
||||
|
||||
def args():
|
||||
'''Read command line arguments and return options.'''
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--channel',
|
||||
help='Release channel to use: '
|
||||
'stable, beta, or nightly',
|
||||
default='stable')
|
||||
parser.add_argument('--cargo-channel',
|
||||
help='Release channel to use for cargo: '
|
||||
'stable, beta, or nightly.'
|
||||
'Defaults to the same as --channel.')
|
||||
args = parser.parse_args()
|
||||
if not args.cargo_channel:
|
||||
args.cargo_channel = args.channel
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = vars(args())
|
||||
repack(mac64, [mac64], **args)
|
||||
repack(win32, [win32], **args)
|
||||
repack(win64, [win64], **args)
|
||||
repack(linux64, [linux64, linux32], **args)
|
||||
repack(linux64, [linux64, mac64], suffix='mac-cross', **args)
|
||||
repack(linux64, [linux64, android, android_x86, android_aarch64],
|
||||
suffix='android-cross', **args)
|
||||
repack(linux64, [linux64, win32, mingw32], suffix='mingw32-cross', **args)
|
||||
@ -1,100 +0,0 @@
|
||||
#!/bin/env python
|
||||
'''
|
||||
This script patches tooltool manifests in the firefox source
|
||||
tree to update them to a new set of rust packages.
|
||||
'''
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import json
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
def load_manifest(path):
|
||||
with open(path) as f:
|
||||
return json.load(f, object_pairs_hook=OrderedDict)
|
||||
return None
|
||||
|
||||
|
||||
def save_manifest(manifest, path):
|
||||
with open(path, 'w') as f:
|
||||
json.dump(manifest, f, indent=2, separators=(',', ': '))
|
||||
f.write('\n')
|
||||
|
||||
|
||||
def replace(manifest, stanza):
|
||||
key = 'rustc'
|
||||
version = stanza.get('version')
|
||||
for s in manifest:
|
||||
if key in s.get('filename'):
|
||||
if version:
|
||||
print('Replacing %s\n with %s' % (s['version'], version))
|
||||
s['version'] = version
|
||||
print(' old %s' % s['digest'][:12])
|
||||
s['digest'] = stanza['digest']
|
||||
s['size'] = stanza['size']
|
||||
print(' new %s' % s['digest'][:12])
|
||||
return True
|
||||
print('Warning: Could not find matching %s filename' % key)
|
||||
return False
|
||||
|
||||
|
||||
def update_manifest(source_manifest, target, target_filename):
|
||||
for stanza in source_manifest:
|
||||
filename = stanza.get('filename')
|
||||
if target in filename:
|
||||
size = int(stanza.get('size'))
|
||||
print('Found %s %d bytes' % (filename, size))
|
||||
version = stanza.get('version')
|
||||
if version:
|
||||
print(' %s' % version)
|
||||
print('Updating %s' % target_filename)
|
||||
old = load_manifest(target_filename)
|
||||
replace(old, stanza)
|
||||
save_manifest(old, target_filename)
|
||||
break
|
||||
|
||||
|
||||
'''Mapping from targets to target filenames.'''
|
||||
TARGETS = {
|
||||
'x86_64-unknown-linux-gnu-repack': [
|
||||
'browser/config/tooltool-manifests/linux32/releng.manifest',
|
||||
'browser/config/tooltool-manifests/linux64/hazard.manifest',
|
||||
'browser/config/tooltool-manifests/linux64/releng.manifest',
|
||||
],
|
||||
'x86_64-unknown-linux-gnu-android-cross-repack': [
|
||||
'mobile/android/config/tooltool-manifests/android/releng.manifest',
|
||||
'mobile/android/config/tooltool-manifests/android-x86/releng.manifest',
|
||||
],
|
||||
'x86_64-unknown-linux-gnu-mingw32-cross-repack': [
|
||||
'browser/config/tooltool-manifests/mingw32/releng.manifest',
|
||||
],
|
||||
'x86_64-unknown-linux-gnu-mac-cross-repack': [
|
||||
'browser/config/tooltool-manifests/macosx64/cross-releng.manifest',
|
||||
],
|
||||
'x86_64-apple-darwin-repack': [
|
||||
'browser/config/tooltool-manifests/macosx64/clang.manifest',
|
||||
'browser/config/tooltool-manifests/macosx64/releng.manifest',
|
||||
],
|
||||
'x86_64-pc-windows-msvc-repack': [
|
||||
'browser/config/tooltool-manifests/win64/releng.manifest',
|
||||
],
|
||||
'i686-pc-windows-msvc-repack': [
|
||||
'browser/config/tooltool-manifests/win32/releng.manifest',
|
||||
],
|
||||
}
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 2:
|
||||
print('%s PATH' % sys.argv[0])
|
||||
sys.exit(1)
|
||||
|
||||
base_path = sys.argv[1]
|
||||
|
||||
updates = load_manifest('manifest.tt')
|
||||
for target, filenames in TARGETS.items():
|
||||
for target_filename in filenames:
|
||||
update_manifest(updates, target,
|
||||
os.path.join(base_path, target_filename))
|
||||
@ -1,37 +0,0 @@
|
||||
{
|
||||
"provisionerId": "aws-provisioner-v1",
|
||||
"workerType": "rustbuild",
|
||||
"created": "{task_created}",
|
||||
"deadline": "{task_deadline}",
|
||||
"payload": {
|
||||
"image": "quay.io/rust/gecko-rust-build",
|
||||
"env": {
|
||||
"RUST_BRANCH": "{rust_branch}"
|
||||
},
|
||||
"artifacts": {
|
||||
"public/rustc.tar.xz": {
|
||||
"path": "/home/worker/rustc.tar.xz",
|
||||
"expires": "{artifact_expires}",
|
||||
"type": "file"
|
||||
},
|
||||
"public/manifest.tt": {
|
||||
"path": "/home/worker/manifest.tt",
|
||||
"expires": "{artifact_expires}",
|
||||
"type": "file"
|
||||
}
|
||||
},
|
||||
"features": {
|
||||
"relengAPIProxy": true
|
||||
},
|
||||
"maxRunTime": 6000
|
||||
},
|
||||
"scopes": [
|
||||
"docker-worker:relengapi-proxy:tooltool.upload.public"
|
||||
],
|
||||
"metadata": {
|
||||
"name": "Rust toolchain build",
|
||||
"description": "Builds the rust toolchain for use in gecko builders.",
|
||||
"owner": "giles@mozilla.com",
|
||||
"source": "https://github.com/rillian/rust-build/"
|
||||
}
|
||||
}
|
||||
@ -1,228 +0,0 @@
|
||||
#!/bin/env python
|
||||
'''
|
||||
This script triggers a taskcluster task, waits for it to finish,
|
||||
fetches the artifacts, uploads them to tooltool, and updates
|
||||
the in-tree tooltool manifests.
|
||||
'''
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import requests.packages.urllib3
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import taskcluster
|
||||
import tempfile
|
||||
import time
|
||||
import tooltool
|
||||
|
||||
|
||||
def local_file(filename):
|
||||
'''
|
||||
Return a path to a file next to this script.
|
||||
'''
|
||||
return os.path.join(os.path.dirname(__file__), filename)
|
||||
|
||||
|
||||
def read_tc_auth(tc_auth_file):
|
||||
'''
|
||||
Read taskcluster credentials from tc_auth_file and return them as a dict.
|
||||
'''
|
||||
return json.load(open(tc_auth_file, 'rb'))
|
||||
|
||||
|
||||
def fill_template_dict(d, keys):
|
||||
for key, val in d.items():
|
||||
if isinstance(val, basestring) and '{' in val:
|
||||
d[key] = val.format(**keys)
|
||||
elif isinstance(val, dict):
|
||||
fill_template_dict(val, keys)
|
||||
|
||||
|
||||
def fill_template(template_file, keys):
|
||||
'''
|
||||
Take the file object template_file, parse it as JSON, and
|
||||
interpolate (using str.template) its keys using keys.
|
||||
'''
|
||||
template = json.load(template_file)
|
||||
fill_template_dict(template, keys)
|
||||
return template
|
||||
|
||||
|
||||
def spawn_task(queue, args):
|
||||
'''
|
||||
Spawn a Taskcluster task in queue using args.
|
||||
'''
|
||||
task_id = taskcluster.utils.slugId()
|
||||
with open(local_file('task.json'), 'rb') as template:
|
||||
keys = vars(args)
|
||||
now = datetime.datetime.utcnow()
|
||||
deadline = (now + datetime.timedelta(hours=2))
|
||||
expires = (now + datetime.timedelta(days=1))
|
||||
keys['task_created'] = now.isoformat() + 'Z'
|
||||
keys['task_deadline'] = deadline.isoformat() + 'Z'
|
||||
keys['artifact_expires'] = expires.isoformat() + 'Z'
|
||||
payload = fill_template(template, keys)
|
||||
queue.createTask(task_id, payload)
|
||||
print('--- %s task %s submitted ---' % (now, task_id))
|
||||
return task_id
|
||||
|
||||
|
||||
def wait_for_task(queue, task_id, initial_wait=5):
|
||||
'''
|
||||
Wait until queue reports that task task_id is completed, and return
|
||||
its run id.
|
||||
|
||||
Sleep for initial_wait seconds before checking status the first time.
|
||||
Then poll periodically and print a running log of the task status.
|
||||
'''
|
||||
time.sleep(initial_wait)
|
||||
previous_state = None
|
||||
have_ticks = False
|
||||
while True:
|
||||
res = queue.status(task_id)
|
||||
state = res['status']['state']
|
||||
if state != previous_state:
|
||||
now = datetime.datetime.utcnow()
|
||||
if have_ticks:
|
||||
sys.stdout.write('\n')
|
||||
have_ticks = False
|
||||
print('--- %s task %s %s ---' % (now, task_id, state))
|
||||
previous_state = state
|
||||
if state == 'completed':
|
||||
return len(res['status']['runs']) - 1
|
||||
if state in ('failed', 'exception'):
|
||||
raise Exception('Task failed')
|
||||
sys.stdout.write('.')
|
||||
sys.stdout.flush()
|
||||
have_ticks = True
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
def fetch_artifact(queue, task_id, run_id, name, dest_dir):
|
||||
'''
|
||||
Fetch the artifact with name from task_id and run_id in queue,
|
||||
write it to a file in dest_dir, and return the path to the written
|
||||
file.
|
||||
'''
|
||||
url = queue.buildUrl('getArtifact', task_id, run_id, name)
|
||||
fn = os.path.join(dest_dir, os.path.basename(name))
|
||||
print('Fetching %s...' % name)
|
||||
try:
|
||||
r = requests.get(url, stream=True)
|
||||
r.raise_for_status()
|
||||
with open(fn, 'wb') as f:
|
||||
for chunk in r.iter_content(1024):
|
||||
f.write(chunk)
|
||||
except requests.exceptions.HTTPError:
|
||||
print('HTTP Error %d fetching %s' % (r.status_code, name))
|
||||
return None
|
||||
return fn
|
||||
|
||||
|
||||
def make_artifact_dir(task_id, run_id):
|
||||
prefix = 'tc-artifacts.%s.%d.' % (task_id, run_id)
|
||||
print('making artifact dir %s' % prefix)
|
||||
return tempfile.mkdtemp(prefix=prefix)
|
||||
|
||||
|
||||
def fetch_artifacts(queue, task_id, run_id):
|
||||
'''
|
||||
Fetch all artifacts from task_id and run_id in queue, write them to
|
||||
temporary files, and yield the path to each.
|
||||
'''
|
||||
try:
|
||||
tempdir = make_artifact_dir(task_id, run_id)
|
||||
res = queue.listArtifacts(task_id, run_id)
|
||||
for a in res['artifacts']:
|
||||
# Skip logs
|
||||
if a['name'].startswith('public/logs'):
|
||||
continue
|
||||
# Skip interfaces
|
||||
if a['name'].startswith('private/docker-worker'):
|
||||
continue
|
||||
yield fetch_artifact(queue, task_id, run_id, a['name'], tempdir)
|
||||
finally:
|
||||
if os.path.isdir(tempdir):
|
||||
# shutil.rmtree(tempdir)
|
||||
print('Artifacts downloaded to %s' % tempdir)
|
||||
pass
|
||||
|
||||
|
||||
def upload_to_tooltool(tooltool_auth, task_id, artifact):
|
||||
'''
|
||||
Upload artifact to tooltool using tooltool_auth as the authentication token.
|
||||
Return the path to the generated tooltool manifest.
|
||||
'''
|
||||
try:
|
||||
oldcwd = os.getcwd()
|
||||
os.chdir(os.path.dirname(artifact))
|
||||
manifest = artifact + '.manifest'
|
||||
tooltool.main([
|
||||
'tooltool.py',
|
||||
'add',
|
||||
'--visibility=public',
|
||||
'-m', manifest,
|
||||
artifact
|
||||
])
|
||||
tooltool.main([
|
||||
'tooltool.py',
|
||||
'upload',
|
||||
'-m', manifest,
|
||||
'--authentication-file', tooltool_auth,
|
||||
'--message', 'Built from taskcluster task {}'.format(task_id),
|
||||
])
|
||||
return manifest
|
||||
finally:
|
||||
os.chdir(oldcwd)
|
||||
|
||||
|
||||
def update_manifest(artifact, manifest, local_gecko_clone):
|
||||
platform = 'linux'
|
||||
manifest_dir = os.path.join(local_gecko_clone,
|
||||
'testing', 'config', 'tooltool-manifests')
|
||||
platform_dir = [p for p in os.listdir(manifest_dir)
|
||||
if p.startswith(platform)][0]
|
||||
tree_manifest = os.path.join(manifest_dir, platform_dir, 'releng.manifest')
|
||||
print('%s -> %s' % (manifest, tree_manifest))
|
||||
shutil.copyfile(manifest, tree_manifest)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Build and upload binaries')
|
||||
parser.add_argument('taskcluster_auth',
|
||||
help='Path to a file containing Taskcluster client '
|
||||
'ID and authentication token as a JSON file in '
|
||||
'the form {"clientId": "...", "accessToken": "..."}')
|
||||
parser.add_argument('--tooltool-auth',
|
||||
help='Path to a file containing a tooltool '
|
||||
'authentication token valid for uploading files')
|
||||
parser.add_argument('--local-gecko-clone',
|
||||
help='Path to a local Gecko clone whose tooltool '
|
||||
'manifests will be updated with the newly-built binaries')
|
||||
parser.add_argument('--rust-branch', default='stable',
|
||||
help='Revision of the rust repository to use')
|
||||
parser.add_argument('--task', help='Use an existing task')
|
||||
|
||||
args = parser.parse_args()
|
||||
tc_auth = read_tc_auth(args.taskcluster_auth)
|
||||
queue = taskcluster.Queue({'credentials': tc_auth})
|
||||
if args.task:
|
||||
task_id, initial_wait = args.task, 0
|
||||
else:
|
||||
task_id, initial_wait = spawn_task(queue, args), 25
|
||||
run_id = wait_for_task(queue, task_id, initial_wait)
|
||||
for artifact in fetch_artifacts(queue, task_id, run_id):
|
||||
if args.tooltool_auth:
|
||||
manifest = upload_to_tooltool(args.tooltool_auth, task_id,
|
||||
artifact)
|
||||
if args.local_gecko_clone:
|
||||
update_manifest(artifact, manifest, args.local_gecko_clone)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,22 +0,0 @@
|
||||
#!/bin/bash -vex
|
||||
|
||||
set -x -e
|
||||
|
||||
: WORKSPACE ${WORKSPACE:=/home/worker}
|
||||
|
||||
set -v
|
||||
|
||||
# Upload artifacts packaged by the build script.
|
||||
pushd ${WORKSPACE}
|
||||
if test -n "$TASK_ID"; then
|
||||
# If we're running on task cluster, use the upload-capable tunnel.
|
||||
TOOLTOOL_OPTS="--url=http://relengapi/tooltool/"
|
||||
MESSAGE="Taskcluster upload ${TASK_ID}/${RUN_ID} $0"
|
||||
else
|
||||
MESSAGE="Rust toolchain build for gecko"
|
||||
fi
|
||||
if test -r rust-version; then
|
||||
MESSAGE="$MESSAGE $(cat rust-version)"
|
||||
fi
|
||||
/build/tooltool.py upload ${TOOLTOOL_OPTS} --message="${MESSAGE}"
|
||||
popd
|
||||
Loading…
Reference in New Issue
Block a user