From d017b0874eac0358e88d2a557a01fcb3facc7db9 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Mon, 18 Sep 2017 15:38:22 -0400 Subject: [PATCH] Bug 1396098 - part 1 - base `android-build` on Debian instead of CentOS; r=glandium,dustin CentOS 6 is pinned to glibc 2.12, but newer Android build-tools (like aapt) require glibc 2.14. It's not possible to safely upgrade CentOS 6 distributions to glibc 2.14. CentOS 7 is pinned to glibc 2.17, which is new enough for newer Android build-tools. However, I had great difficulty bringing forward our existing centos:6 Docker image to centos:7. In particular, installing recent enough Mercurial, git, Python, and pip versions was difficult enough that I elected to not pursue this approach. Instead, I've elected to follow glandium's suggestion from https://bugzilla.mozilla.org/show_bug.cgi?id=1370119#c5: base on Debian with snapshots.debian.org for reproducibility. The most significant changes here: - using Debian's snapshots repository - using Python and related tools provided by Debian and baked into the build image - using the JDK and JRE provided by Debian and baked into the build image, rather than versions from tooltool (or eventually a toolchain build) Moving the builds over to use this image will follow in the patches ahead. --- taskcluster/docker/android-build/Dockerfile | 153 ++++++++++++------ .../docker/android-build/buildprops.json | 9 -- taskcluster/docker/android-build/oauth.txt | 2 - taskcluster/docker/android-build/sources.list | 3 + .../recipes/debian-build-system-setup.sh | 11 ++ .../docker/recipes/install-mercurial.sh | 17 ++ .../build-android-dependencies/after.sh | 11 +- .../build-android-dependencies/before.sh | 23 ++- 8 files changed, 144 insertions(+), 85 deletions(-) delete mode 100644 taskcluster/docker/android-build/buildprops.json delete mode 100644 taskcluster/docker/android-build/oauth.txt create mode 100644 taskcluster/docker/android-build/sources.list create mode 100644 taskcluster/docker/recipes/debian-build-system-setup.sh mode change 100644 => 100755 taskcluster/docker/recipes/install-mercurial.sh mode change 100644 => 100755 taskcluster/scripts/builder/build-android-dependencies/after.sh mode change 100644 => 100755 taskcluster/scripts/builder/build-android-dependencies/before.sh diff --git a/taskcluster/docker/android-build/Dockerfile b/taskcluster/docker/android-build/Dockerfile index c3da2ad9e9ba..c5a9a24e2b19 100644 --- a/taskcluster/docker/android-build/Dockerfile +++ b/taskcluster/docker/android-build/Dockerfile @@ -1,18 +1,90 @@ -# TODO remove VOLUME below when the base image is updated next. -FROM taskcluster/centos6-build-upd:0.1.8.20170808150401 -MAINTAINER Nick Alexander +FROM debian:stretch-20170620 +MAINTAINER Nick Alexander -# BEGIN ../desktop-build/Dockerfile +### Add worker user and setup its workspace. +RUN mkdir -p /builds && \ + groupadd -g 500 worker && \ + useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker && \ + chown -R worker:worker /builds && \ + mkdir -p /builds/worker/workspace && \ + chown -R worker:worker /builds/worker/workspace + +# Declare default working folder +WORKDIR /builds/worker VOLUME /builds/worker/checkouts VOLUME /builds/worker/workspace VOLUME /builds/worker/tooltool-cache +# Add build scripts; these are the entry points from the taskcluster worker, and +# operate on environment variables +# %include taskcluster/docker/desktop-build/bin/ +COPY topsrcdir/taskcluster/docker/desktop-build/bin/ /builds/worker/bin/ +RUN chmod +x /builds/worker/bin/* + +# Add common hg configuration. +# %include taskcluster/docker/centos6-build/hgrc +COPY topsrcdir/taskcluster/docker/centos6-build/hgrc /builds/worker/.hgrc +RUN chown worker:worker /builds/worker/.hgrc + +# Set variables normally configured at login, by the shell' parent process, +# these are taken from GNU su manual. Also set DEBIAN_FRONTEND to advise +# apt-get of our situation. +ENV HOME=/builds/worker \ + SHELL=/bin/bash \ + USER=worker \ + LOGNAME=worker \ + HOSTNAME=taskcluster-worker \ + DEBIAN_FRONTEND=noninteractive + +# Set a default command useful for debugging. +CMD ["/bin/bash", "--login"] + +# Set apt sources list to a snapshot. +COPY sources.list /etc/apt/ + +# We need i386 packages for the Android SDK. +# Once https://bugzilla.mozilla.org/show_bug.cgi?id=1370119 is in-tree, we +# will have 64-bit builds of everything, and we can then remove this and +# the :i386 packages we install below. +RUN dpkg --add-architecture i386 + +RUN apt-get update -q && \ + apt-get install -yyq --no-install-recommends \ + autoconf2.13 \ + build-essential \ + base-files \ + ca-certificates \ + ccache \ + cmake \ + curl \ + file \ + gnupg \ + make \ + procps \ + python \ + python-cryptography \ + python-dev \ + python-pip \ + python-setuptools \ + python-virtualenv \ + sudo \ + tar \ + unzip \ + uuid \ + wget \ + xz-utils \ + yasm \ + zip \ + libstdc++6:i386 \ + libgcc1:i386 \ + zlib1g:i386 \ + libncurses5:i386 \ + && \ + apt-get clean + # %include python/mozbuild/mozbuild/action/tooltool.py -COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /build/tooltool.py -COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/tooltool.py COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py -COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /tmp/tooltool.py # %include testing/mozharness/external_tools/robustcheckout.py COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py @@ -29,70 +101,52 @@ COPY topsrcdir/taskcluster/docker/recipes/install-make.sh /setup/install-make.sh # %include taskcluster/docker/recipes/install-cmake.sh COPY topsrcdir/taskcluster/docker/recipes/install-cmake.sh /setup/install-cmake.sh -# %include taskcluster/docker/recipes/centos6-build-system-setup.sh -COPY topsrcdir/taskcluster/docker/recipes/centos6-build-system-setup.sh /setup/system-setup.sh - -# %include taskcluster/docker/recipes/centos-install-debug-symbols.sh -COPY topsrcdir/taskcluster/docker/recipes/centos-install-debug-symbols.sh /setup/install-debug-symbols.sh - -# TODO remove once base image doesn't install Mercurial -RUN pip uninstall -y Mercurial +# %include taskcluster/docker/recipes/debian-build-system-setup.sh +COPY topsrcdir/taskcluster/docker/recipes/debian-build-system-setup.sh /setup/system-setup.sh RUN bash /setup/system-setup.sh -# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb +# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb. # %include taskcluster/docker/recipes/xvfb.sh COPY topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh # %include taskcluster/docker/recipes/run-task COPY topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task -# Add configuration -COPY dot-config /builds/worker/.config - -# Generate machine uuid file -RUN dbus-uuidgen --ensure=/var/lib/dbus/machine-id +# Add pip configuration, among other things. +COPY dot-config /builds/worker/.config # Stubbed out credentials; mozharness looks for this file an issues a WARNING # if it's not found, which causes the build to fail. Note that this needs to # be in the parent of the workspace directory and in the directory where # mozharness is run (not its --work-dir). See Bug 1169652. -ADD oauth.txt /builds/worker/ +# %include taskcluster/docker/desktop-build/oauth.txt +COPY topsrcdir/taskcluster/docker/desktop-build/oauth.txt /builds/worker/ -# stubbed out buildprops, which keeps mozharness from choking +# Add a stubbed out buildprops, which keeps mozharness from choking. # Note that this needs to be in the parent of the workspace directory and in -# the directory where mozharness is run (not its --work-dir) -ADD buildprops.json /builds/worker/ - -# END ../desktop-build/Dockerfile +# the directory where mozharness is run (not its --work-dir). +# %include taskcluster/docker/desktop-build/buildprops.json +COPY topsrcdir/taskcluster/docker/desktop-build/buildprops.json /builds/worker/ # Reset user/workdir from parent image so we can install software. WORKDIR / USER root -# Update base. -RUN rpm --rebuilddb && yum upgrade -y - # Install Sonatype Nexus. Cribbed directly from # https://github.com/sonatype/docker-nexus/blob/fffd2c61b2368292040910c055cf690c8e76a272/oss/Dockerfile. -# Install the screen package here to use with xvfb. -# Move installation to base centos6-build image once Bug 1272629 is fixed -# tzdata-java is symlinked from the JRE: see https://bugzilla.redhat.com/show_bug.cgi?id=1130800 -RUN rpm --rebuilddb && yum install -y \ - createrepo \ - curl \ - screen \ - sudo \ - tar \ - tzdata-java \ - unzip \ - wget \ - zip \ - && yum clean all +# Observe missing --no-install-recommends, in order to install glib2.0/gconf/etc. +RUN apt-get install -yyq \ + libgconf2-4 \ + libgnome2-0 \ + openjdk-8-jdk-headless \ + && \ + apt-get clean -ENV NEXUS_VERSION 2.12.0-01 -ENV NEXUS_SHA1SUM 1a9aaad8414baffe0a2fd46eed1f41b85f4049e6 +ENV NEXUS_VERSION=2.12.0-01 \ + NEXUS_SHA1SUM=1a9aaad8414baffe0a2fd46eed1f41b85f4049e6 \ + NEXUS_WORK=/builds/worker/workspace/nexus RUN mkdir -p /opt/sonatype/nexus @@ -111,10 +165,7 @@ RUN tar zxf nexus-${NEXUS_VERSION}-bundle.tar.gz \ && rm -rf /tmp/nexus-${NEXUS_VERSION}-bundle.tar.gz # So that we don't have to RUN_AS_USER=root. -RUN chown -R worker:worker /opt/sonatype/nexus/ +RUN chown -R worker:worker /opt/sonatype -# Back to the centos6-build workdir, matching desktop-build. +# Back to the workdir, matching desktop-build. WORKDIR /builds/worker - -# Set a default command useful for debugging -CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/android-build/buildprops.json b/taskcluster/docker/android-build/buildprops.json deleted file mode 100644 index f38b7d788e0b..000000000000 --- a/taskcluster/docker/android-build/buildprops.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "properties": { - "buildername": "" - }, - "sourcestamp": { - "changes": [] - }, - "comments": "TaskCluster Job" -} diff --git a/taskcluster/docker/android-build/oauth.txt b/taskcluster/docker/android-build/oauth.txt deleted file mode 100644 index e56c71f577cc..000000000000 --- a/taskcluster/docker/android-build/oauth.txt +++ /dev/null @@ -1,2 +0,0 @@ -taskcluster_clientId = None -taskcluster_accessToken = None diff --git a/taskcluster/docker/android-build/sources.list b/taskcluster/docker/android-build/sources.list new file mode 100644 index 000000000000..41987f8e8dc5 --- /dev/null +++ b/taskcluster/docker/android-build/sources.list @@ -0,0 +1,3 @@ +deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/20170830T000511Z/ stretch main +deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/20170830T000511Z/ stretch-updates main +deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/20170830T000511Z/ stretch/updates main diff --git a/taskcluster/docker/recipes/debian-build-system-setup.sh b/taskcluster/docker/recipes/debian-build-system-setup.sh new file mode 100644 index 000000000000..bf1d2c78aed0 --- /dev/null +++ b/taskcluster/docker/recipes/debian-build-system-setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cd /setup + +. /setup/common.sh +. /setup/install-mercurial.sh + +rm -rf /setup diff --git a/taskcluster/docker/recipes/install-mercurial.sh b/taskcluster/docker/recipes/install-mercurial.sh old mode 100644 new mode 100755 index 0c977225caff..5b5d55dbded1 --- a/taskcluster/docker/recipes/install-mercurial.sh +++ b/taskcluster/docker/recipes/install-mercurial.sh @@ -27,6 +27,23 @@ if [ -f /etc/lsb-release ]; then CERT_PATH=/etc/ssl/certs/ca-certificates.crt +elif [ -f /etc/os-release ]; then + . /etc/os-release + + if [ "${ID}" = "debian" -a "${VERSION_ID}" = "9" ]; then + if [ -f /usr/bin/pip2 ]; then + PIP_PATH=/usr/bin/pip2 + else + echo "We currently require Python 2.7 and /usr/bin/pip2 to run Mercurial" + exit 1 + fi + else + echo "Unsupported debian-like system with ID '${ID}' and VERSION_ID '${VERSION_ID}'" + exit 1 + fi + + CERT_PATH=/etc/ssl/certs/ca-certificates.crt + elif [ -f /etc/centos-release ]; then CENTOS_VERSION=`rpm -q --queryformat '%{VERSION}' centos-release` if [ "${CENTOS_VERSION}" = "6" ]; then diff --git a/taskcluster/scripts/builder/build-android-dependencies/after.sh b/taskcluster/scripts/builder/build-android-dependencies/after.sh old mode 100644 new mode 100755 index b11a2590d520..16e218c190d4 --- a/taskcluster/scripts/builder/build-android-dependencies/after.sh +++ b/taskcluster/scripts/builder/build-android-dependencies/after.sh @@ -19,17 +19,10 @@ tar cJf android-sdk-linux.tar.xz android-sdk-linux mkdir -p /builds/worker/private/android-sdk mv android-sdk-linux.tar.xz /builds/worker/private/android-sdk -cp -R $WORKSPACE/build/src/java_home java_home -tar cJf java_home.tar.xz java_home - -# We can't redistribute Java publicly. -mkdir -p /builds/worker/private/java_home -mv java_home.tar.xz /builds/worker/private/java_home - -cp -R $WORKSPACE/nexus/storage/jcenter jcenter +cp -R ${NEXUS_WORK}/storage/jcenter jcenter tar cJf jcenter.tar.xz jcenter -cp -R $WORKSPACE/nexus/storage/google google +cp -R ${NEXUS_WORK}/storage/google google tar cJf google.tar.xz google # The Gradle wrapper will have downloaded and verified the hash of exactly one diff --git a/taskcluster/scripts/builder/build-android-dependencies/before.sh b/taskcluster/scripts/builder/build-android-dependencies/before.sh old mode 100644 new mode 100755 index 46df2a437a18..79942ffdbb9f --- a/taskcluster/scripts/builder/build-android-dependencies/before.sh +++ b/taskcluster/scripts/builder/build-android-dependencies/before.sh @@ -8,21 +8,10 @@ echo "running as" $(id) set -v -mkdir -p ${WORKSPACE}/nexus/conf -cp /builds/worker/workspace/build/src/taskcluster/scripts/builder/build-android-dependencies/nexus.xml ${WORKSPACE}/nexus/conf/nexus.xml +mkdir -p ${NEXUS_WORK}/conf +cp /builds/worker/workspace/build/src/taskcluster/scripts/builder/build-android-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml -# Populate /builds/worker/workspace/build/src/java_home. -. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh - -mv $WORKSPACE/java/usr/lib/jvm/java_home $WORKSPACE/build/src/java_home - -export JAVA_HOME=$WORKSPACE/build/src/java_home -export PATH=$PATH:$JAVA_HOME/bin - -# Populate /builds/worker/.mozbuild/android-sdk-linux. -python2.7 $WORKSPACE/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive - -RUN_AS_USER=worker NEXUS_WORK=$WORKSPACE/nexus /opt/sonatype/nexus/bin/nexus restart +RUN_AS_USER=worker /opt/sonatype/nexus/bin/nexus restart # Wait "a while" for Nexus to actually start. Don't fail if this fails. wget --quiet --retry-connrefused --waitretry=2 --tries=100 \ @@ -34,3 +23,9 @@ curl http://localhost:8081/nexus/service/local/status || true # Verify Nexus has actually started. Fail if this fails. curl --fail --silent --location http://localhost:8081/nexus/service/local/status | grep 'STARTED' + +# It's helpful when debugging to see the repository configurations. +curl http://localhost:8081/nexus/service/local/repositories || true + +# Populate /builds/worker/.mozbuild/android-sdk-linux. +python2.7 /builds/worker/workspace/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive