Bug 1289605 - Update gecko-rust-build to v0.3.0. r=dustin

- Separate build, package, and upload steps into separate scripts.
- Rewrite repack in python to use rust's internal install scripts.
- Add support for cargo as well as the rust compiler and std library.

MozReview-Commit-ID: EI9M8ayEptA

--HG--
extra : rebase_source : 4457c074396e5c733158d304c2aebe0f811a9d7e

testing/docker/rust-build/Dockerfile

@@ -15,8 +15,10 @@ ADD https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py
 RUN chmod +rx /build/tooltool.py
 
 # Add build scripts.
-ADD             fetch_rust.sh build_rust.sh upload_rust.sh /build/
-ADD             repack_rust.sh /build/
+ADD             fetch_rust.sh build_rust.sh /build/
+ADD             fetch_cargo.sh build_cargo.sh /build/
+ADD             package_rust.sh upload_rust.sh /build/
+ADD             repack_rust.py /build/
 RUN             chmod +x /build/*
 
 # Create user for doing the build.
@@ -30,4 +32,6 @@ WORKDIR ${HOME}
 
 # Invoke our build scripts by default, but allow other commands.
 USER ${USER}
-ENTRYPOINT /build/fetch_rust.sh && /build/build_rust.sh && /build/upload_rust.sh
+ENTRYPOINT /build/fetch_rust.sh && /build/build_rust.sh && \
+  /build/fetch_cargo.sh && /build/build_cargo.sh && \
+  /build/package_rust.sh && /build/upload_rust.sh

testing/docker/rust-build/VERSION

@@ -1 +1 @@
-0.2.1
+0.3.0

testing/docker/rust-build/build_cargo.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -vex
+
+set -x -e
+
+: WORKSPACE ${WORKSPACE:=/home/worker}
+
+set -v
+
+# Configure and build cargo.
+
+if test $(uname -s) = "Darwin"; then
+  export MACOSX_DEPLOYMENT_TARGET=10.7
+fi
+
+pushd ${WORKSPACE}/cargo
+./configure --prefix=${WORKSPACE}/rustc --local-rust-root=${WORKSPACE}/rustc
+make
+make dist
+make install
+popd

testing/docker/rust-build/build_rust.sh

@@ -9,7 +9,9 @@ CORES=$(nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu)
 set -v
 
 # Configure and build rust.
-OPTIONS="--enable-llvm-static-stdcpp --disable-docs --release-channel=stable"
+OPTIONS="--enable-llvm-static-stdcpp --disable-docs"
+OPTIONS+="--enable-debuginfo"
+OPTIONS+="--release-channel=stable"
 x32="i686-unknown-linux-gnu"
 x64="x86_64-unknown-linux-gnu"
 arm="arm-linux-androideabi"
@@ -22,9 +24,3 @@ make -j ${CORES}
 make dist
 make install
 popd
-
-# Package the toolchain for upload.
-pushd ${WORKSPACE}
-tar cvJf rustc.tar.xz rustc/*
-/build/tooltool.py add --visibility=public --unpack rustc.tar.xz
-popd

testing/docker/rust-build/build_rust_mac.sh

@@ -1,15 +1,20 @@
 #!/bin/bash -vex
 
+set -e
+
 : WORKSPACE ${WORKSPACE:=$PWD}
+: TOOLTOOL ${TOOLTOOL:=python $WORKSPACE/tooltool.py}
 
 CORES=$(nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu)
 echo Building on $CORES cpus...
 
-OPTIONS="--disable-elf-tls --disable-docs"
+OPTIONS="--enable-debuginfo --disable-docs"
 TARGETS="x86_64-apple-darwin,i686-apple-darwin"
 
 PREFIX=${WORKSPACE}/rustc
 
+set -v
+
 mkdir -p ${WORKSPACE}/gecko-rust-mac
 pushd ${WORKSPACE}/gecko-rust-mac
 
@@ -25,6 +30,7 @@ popd
 
 # Package the toolchain for upload.
 pushd ${WORKSPACE}
+rustc/bin/rustc --version
 tar cvjf rustc.tar.bz2 rustc/*
-python tooltool.py add --visibility=public --unpack rustc.tar.bz2
+${TOOLTOOL} add --visibility=public --unpack rustc.tar.bz2
 popd

testing/docker/rust-build/fetch_cargo.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -vex
+
+set -x -e
+
+# Inputs, with defaults
+
+: REPOSITORY   ${REPOSITORY:=https://github.com/rust-lang/cargo}
+: BRANCH       ${BRANCH:=master}
+
+: WORKSPACE    ${WORKSPACE:=/home/worker}
+
+set -v
+
+# Check out rust sources
+SRCDIR=${WORKSPACE}/cargo
+git clone --recursive $REPOSITORY -b $BRANCH ${SRCDIR}
+
+# Report version
+VERSION=$(git -C ${SRCDIR} describe --tags --dirty)
+COMMIT=$(git -C ${SRCDIR} rev-parse HEAD)
+echo "cargo ${VERSION} (commit ${COMMIT})" | tee cargo-version

testing/docker/rust-build/package_rust.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -vex
+
+set -x -e
+
+: WORKSPACE ${WORKSPACE:=/home/worker}
+
+set -v
+
+# Package the toolchain for upload.
+pushd ${WORKSPACE}
+tar cvJf rustc.tar.xz rustc/*
+/build/tooltool.py add --visibility=public --unpack rustc.tar.xz
+popd

testing/docker/rust-build/repack_rust.py

@@ -0,0 +1,177 @@
+#!/bin/env python
+'''
+This script downloads and repacks official rust language builds
+with the necessary tool and target support for the Firefox
+build environment.
+'''
+
+import os.path
+import requests
+import subprocess
+import toml
+
+def fetch_file(url):
+  '''Download a file from the given url if it's not already present.'''
+  filename = os.path.basename(url)
+  if os.path.exists(filename):
+    return
+  r = requests.get(url, stream=True)
+  r.raise_for_status()
+  with open(filename, 'wb') as fd:
+    for chunk in r.iter_content(4096):
+      fd.write(chunk)
+
+def fetch(url):
+  '''Download and verify a package url.'''
+  base = os.path.basename(url)
+  print('Fetching %s...' % base)
+  fetch_file(url + '.asc')
+  fetch_file(url)
+  fetch_file(url + '.sha256')
+  fetch_file(url + '.asc.sha256')
+  print('Verifying %s...' % base)
+  subprocess.check_call(['shasum', '-c', base + '.sha256'])
+  subprocess.check_call(['shasum', '-c', base + '.asc.sha256'])
+  subprocess.check_call(['gpg', '--verify', base + '.asc', base])
+  subprocess.check_call(['keybase', 'pgp', 'verify',
+      '-d', base + '.asc',
+      '-i', base,
+  ])
+
+def install(filename, target):
+  '''Run a package's installer script against the given target directory.'''
+  print(' Unpacking %s...' % filename)
+  subprocess.check_call(['tar', 'xf', filename])
+  basename = filename.split('.tar')[0]
+  print(' Installing %s...' % basename)
+  install_cmd = [os.path.join(basename, 'install.sh')]
+  install_cmd += ['--prefix=' + os.path.abspath(target)]
+  install_cmd += ['--disable-ldconfig']
+  subprocess.check_call(install_cmd)
+  print(' Cleaning %s...' % basename)
+  subprocess.check_call(['rm', '-rf', basename])
+
+def package(manifest, pkg, target):
+  '''Pull out the package dict for a particular package and target
+  from the given manifest.'''
+  version = manifest['pkg'][pkg]['version']
+  info = manifest['pkg'][pkg]['target'][target]
+  return (version, info)
+
+def fetch_package(manifest, pkg, host):
+  version, info = package(manifest, pkg, host)
+  print('%s %s\n  %s\n  %s' % (pkg, version, info['url'], info['hash']))
+  if not info['available']:
+    print('%s marked unavailable for %s' % (pkg, host))
+    raise AssertionError
+  fetch(info['url'])
+  return info
+
+def fetch_std(manifest, targets):
+  stds = []
+  for target in targets:
+      info = fetch_package(manifest, 'rust-std', target)
+      stds.append(info)
+  return stds
+
+def tar_for_host(host):
+  if 'linux' in host:
+      tar_options = 'cJf'
+      tar_ext = '.tar.xz'
+  else:
+      tar_options = 'cjf'
+      tar_ext = '.tar.bz2'
+  return tar_options, tar_ext
+
+def repack(host, targets, channel='stable', suffix=''):
+  print("Repacking rust for %s..." % host)
+  url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
+  req = requests.get(url)
+  req.raise_for_status()
+  manifest = toml.loads(req.content)
+  if manifest['manifest-version'] != '2':
+    print('ERROR: unrecognized manifest version %s.' % manifest['manifest-version'])
+    return
+  print('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
+  print('Fetching packages...')
+  rustc = fetch_package(manifest, 'rustc', host)
+  cargo = fetch_package(manifest, 'cargo', host)
+  stds = fetch_std(manifest, targets)
+  print('Installing packages...')
+  tar_basename = 'rustc-' + host
+  if suffix:
+      tar_basename += '-' + suffix
+  tar_basename += '-repack'
+  install_dir = 'rustc'
+  subprocess.check_call(['rm', '-rf', install_dir])
+  install(os.path.basename(rustc['url']), install_dir)
+  install(os.path.basename(cargo['url']), install_dir)
+  for std in stds:
+    install(os.path.basename(std['url']), install_dir)
+    pass
+  print('Tarring %s...' % tar_basename)
+  tar_options, tar_ext = tar_for_host(host)
+  subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir])
+  subprocess.check_call(['rm', '-rf', install_dir])
+
+def repack_cargo(host, channel='nightly'):
+  print("Repacking cargo for %s..." % host)
+  # Cargo doesn't seem to have a .toml manifest.
+  base_url = 'https://static.rust-lang.org/cargo-dist/'
+  req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel))
+  req.raise_for_status()
+  file = ''
+  for line in req.iter_lines():
+      if line.find(host) != -1:
+          file = line.strip()
+  if not file:
+      print('No manifest entry for %s!' % host)
+      return
+  manifest = {
+          'date': req.headers['Last-Modified'],
+          'pkg': {
+              'cargo': {
+                  'version': channel,
+                  'target': {
+                      host: {
+                          'url': os.path.join(base_url, file),
+                          'hash': None,
+                          'available': True,
+                      },
+                  },
+              },
+          },
+  }
+  print('Using manifest for cargo %s.' % channel)
+  print('Fetching packages...')
+  cargo = fetch_package(manifest, 'cargo', host)
+  print('Installing packages...')
+  install_dir = 'cargo'
+  subprocess.check_call(['rm', '-rf', install_dir])
+  install(os.path.basename(cargo['url']), install_dir)
+  tar_basename = 'cargo-%s-repack' % host
+  print('Tarring %s...' % tar_basename)
+  tar_options, tar_ext = tar_for_host(host)
+  subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir])
+  subprocess.check_call(['rm', '-rf', install_dir])
+
+# rust platform triples
+android="armv7-linux-androideabi"
+linux64="x86_64-unknown-linux-gnu"
+linux32="i686-unknown-linux-gnu"
+mac64="x86_64-apple-darwin"
+mac32="i686-apple-darwin"
+win64="x86_64-pc-windows-msvc"
+win32="i686-pc-windows-msvc"
+
+if __name__ == '__main__':
+  repack(mac64, [mac64, mac32])
+  repack(win32, [win32])
+  repack(win64, [win64])
+  repack(linux64, [linux64, linux32])
+  repack(linux64, [linux64, mac64, mac32], suffix='mac-cross')
+  repack(linux64, [linux64, android], suffix='android-cross')
+  repack_cargo(mac64)
+  repack_cargo(win32)
+  repack_cargo(win64)
+  repack_cargo(linux64)

testing/docker/rust-build/repack_rust.sh

@@ -1,160 +0,0 @@
-#!/bin/bash -vex
-
-set -e
-
-# Set verbose options on taskcluster for logging.
-test -n "$TASK_ID" && set -x
-
-# Inputs, with defaults
-
-: RUST_URL        ${RUST_URL:=https://static.rust-lang.org/dist/}
-: RUST_CHANNEL    ${RUST_CHANNEL:=stable}
-
-: WORKSPACE       ${WORKSPACE:=/home/worker}
-
-die() {
-  echo "ERROR: $@"
-  exit 1
-}
-
-fetch() {
-  echo "Fetching $1..."
-  curl -Os ${RUST_URL}${1}.asc
-  curl -Os ${RUST_URL}${1}
-  curl -Os ${RUST_URL}${1}.sha256
-  curl -Os ${RUST_URL}${1}.asc.sha256
-}
-
-verify() {
-  echo "Verifying $1..."
-  shasum -c ${1}.sha256
-  shasum -c ${1}.asc.sha256
-  gpg --verify ${1}.asc ${1}
-  keybase verify ${1}.asc
-}
-
-fetch_rustc() {
-  arch=$1
-  echo "Retrieving rustc build for $arch..."
-  pkg=$(cat ${IDX} | grep ^rustc | grep $arch)
-  test -n "${pkg}" || die "No rustc build for $arch in the manifest."
-  test 1 == $(echo ${pkg} | wc -w) ||
-    die "Multiple rustc builds for $arch in the manifest."
-  fetch ${pkg}
-  verify ${pkg}
-}
-
-fetch_std() {
-  echo "Retrieving rust-std builds for:"
-  for arch in $@; do
-    echo "  $arch"
-  done
-  for arch in $@; do
-    pkg=$(cat ${IDX} | grep rust-std | grep $arch)
-    test -n "${pkg}" || die "No rust-std builds for $arch in the manifest."
-    test 1 == $(echo ${pkg} | wc -w) ||
-      die "Multiple rust-std builds for $arch in the manifest."
-    fetch ${pkg}
-    verify ${pkg}
-  done
-}
-
-install_rustc() {
-  pkg=$(cat ${IDX} | grep ^rustc | grep $1)
-  base=${pkg%%.tar.*}
-  echo "Installing $base..."
-  tar xf ${pkg}
-  ${base}/install.sh ${INSTALL_OPTS}
-  rm -rf ${base}
-}
-
-install_std() {
-  for arch in $@; do
-    for pkg in $(cat ${IDX} | grep rust-std | grep $arch); do
-      base=${pkg%%.tar.*}
-      echo "Installing $base..."
-      tar xf ${pkg}
-      ${base}/install.sh ${INSTALL_OPTS}
-      rm -rf ${base}
-    done
-  done
-}
-
-check() {
-  if test -x ${TARGET}/bin/rustc; then
-    file ${TARGET}/bin/rustc
-    ${TARGET}/bin/rustc --version
-  elif test -x ${TARGET}/bin/rustc.exe; then
-    file ${TARGET}/bin/rustc.exe
-    ${TARGET}/bin/rustc.exe --version
-  else
-    die "ERROR: Couldn't fine rustc executable"
-  fi
-  echo "Installed components:"
-  for component in $(cat ${TARGET}/lib/rustlib/components); do
-    echo "  $component"
-  done
-  echo
-}
-
-test -n "$TASK_ID" && set -v
-
-linux64="x86_64-unknown-linux-gnu"
-linux32="i686-unknown-linux-gnu"
-
-android="arm-linux-androideabi"
-
-win64="x86_64-pc-windows-msvc"
-win32="i686-pc-windows-msvc"
-win32_i586="i586-pc-windows-msvc"
-
-# Fetch the manifest
-
-IDX=channel-rustc-${RUST_CHANNEL}
-
-fetch ${IDX}
-verify ${IDX}
-
-TARGET=rustc
-INSTALL_OPTS="--prefix=${PWD}/${TARGET} --disable-ldconfig"
-
-# Repack the linux64 builds.
-
-fetch_rustc $linux64
-fetch_std $linux64 $linux32
-
-rm -rf ${TARGET}
-
-install_rustc $linux64
-install_std $linux64 $linux32
-
-tar cJf rustc-$linux64-repack.tar.xz ${TARGET}/*
-check ${TARGET}
-
-# Repack the win64 builds.
-
-fetch_rustc $win64
-fetch_std $win64
-
-rm -rf ${TARGET}
-
-install_rustc $win64
-install_std $win64
-
-tar cjf rustc-$win64-repack.tar.bz2 ${TARGET}/*
-check ${TARGET}
-
-# Repack the win32 builds.
-
-fetch_rustc $win32
-fetch_std $win32 $win32_i586
-
-rm -rf ${TARGET}
-
-install_rustc $win32
-install_std $win32 $win32_i586
-
-tar cjf rustc-$win32-repack.tar.bz2 ${TARGET}/*
-check ${TARGET}
-
-rm -rf ${TARGET}