mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-15 06:15:43 +00:00
Bug 1396098 - part 1 - base android-build on Debian instead of CentOS; r=glandium,dustin
CentOS 6 is pinned to glibc 2.12, but newer Android build-tools (like aapt) require glibc 2.14. It's not possible to safely upgrade CentOS 6 distributions to glibc 2.14. CentOS 7 is pinned to glibc 2.17, which is new enough for newer Android build-tools. However, I had great difficulty bringing forward our existing centos:6 Docker image to centos:7. In particular, installing recent enough Mercurial, git, Python, and pip versions was difficult enough that I elected to not pursue this approach. Instead, I've elected to follow glandium's suggestion from https://bugzilla.mozilla.org/show_bug.cgi?id=1370119#c5: base on Debian with snapshots.debian.org for reproducibility. The most significant changes here: - using Debian's snapshots repository - using Python and related tools provided by Debian and baked into the build image - using the JDK and JRE provided by Debian and baked into the build image, rather than versions from tooltool (or eventually a toolchain build) Moving the builds over to use this image will follow in the patches ahead.
This commit is contained in:
Nick Alexander
parent
2b102d3c56
commit
d017b0874e
@ -1,18 +1,90 @@
|
||||
# TODO remove VOLUME below when the base image is updated next.
|
||||
FROM taskcluster/centos6-build-upd:0.1.8.20170808150401
|
||||
FROM debian:stretch-20170620
|
||||
MAINTAINER Nick Alexander <nalexander@mozilla.com>
|
||||
|
||||
# BEGIN ../desktop-build/Dockerfile
|
||||
### Add worker user and setup its workspace.
|
||||
RUN mkdir -p /builds && \
|
||||
groupadd -g 500 worker && \
|
||||
useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker && \
|
||||
chown -R worker:worker /builds && \
|
||||
mkdir -p /builds/worker/workspace && \
|
||||
chown -R worker:worker /builds/worker/workspace
|
||||
|
||||
# Declare default working folder
|
||||
WORKDIR /builds/worker
|
||||
|
||||
VOLUME /builds/worker/checkouts
|
||||
VOLUME /builds/worker/workspace
|
||||
VOLUME /builds/worker/tooltool-cache
|
||||
|
||||
# Add build scripts; these are the entry points from the taskcluster worker, and
|
||||
# operate on environment variables
|
||||
# %include taskcluster/docker/desktop-build/bin/
|
||||
COPY topsrcdir/taskcluster/docker/desktop-build/bin/ /builds/worker/bin/
|
||||
RUN chmod +x /builds/worker/bin/*
|
||||
|
||||
# Add common hg configuration.
|
||||
# %include taskcluster/docker/centos6-build/hgrc
|
||||
COPY topsrcdir/taskcluster/docker/centos6-build/hgrc /builds/worker/.hgrc
|
||||
RUN chown worker:worker /builds/worker/.hgrc
|
||||
|
||||
# Set variables normally configured at login, by the shell' parent process,
|
||||
# these are taken from GNU su manual. Also set DEBIAN_FRONTEND to advise
|
||||
# apt-get of our situation.
|
||||
ENV HOME=/builds/worker \
|
||||
SHELL=/bin/bash \
|
||||
USER=worker \
|
||||
LOGNAME=worker \
|
||||
HOSTNAME=taskcluster-worker \
|
||||
DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Set a default command useful for debugging.
|
||||
CMD ["/bin/bash", "--login"]
|
||||
|
||||
# Set apt sources list to a snapshot.
|
||||
COPY sources.list /etc/apt/
|
||||
|
||||
# We need i386 packages for the Android SDK.
|
||||
# Once https://bugzilla.mozilla.org/show_bug.cgi?id=1370119 is in-tree, we
|
||||
# will have 64-bit builds of everything, and we can then remove this and
|
||||
# the :i386 packages we install below.
|
||||
RUN dpkg --add-architecture i386
|
||||
|
||||
RUN apt-get update -q && \
|
||||
apt-get install -yyq --no-install-recommends \
|
||||
autoconf2.13 \
|
||||
build-essential \
|
||||
base-files \
|
||||
ca-certificates \
|
||||
ccache \
|
||||
cmake \
|
||||
curl \
|
||||
file \
|
||||
gnupg \
|
||||
make \
|
||||
procps \
|
||||
python \
|
||||
python-cryptography \
|
||||
python-dev \
|
||||
python-pip \
|
||||
python-setuptools \
|
||||
python-virtualenv \
|
||||
sudo \
|
||||
tar \
|
||||
unzip \
|
||||
uuid \
|
||||
wget \
|
||||
xz-utils \
|
||||
yasm \
|
||||
zip \
|
||||
libstdc++6:i386 \
|
||||
libgcc1:i386 \
|
||||
zlib1g:i386 \
|
||||
libncurses5:i386 \
|
||||
&& \
|
||||
apt-get clean
|
||||
|
||||
# %include python/mozbuild/mozbuild/action/tooltool.py
|
||||
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /build/tooltool.py
|
||||
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/tooltool.py
|
||||
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
|
||||
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /tmp/tooltool.py
|
||||
|
||||
# %include testing/mozharness/external_tools/robustcheckout.py
|
||||
COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
|
||||
@ -29,70 +101,52 @@ COPY topsrcdir/taskcluster/docker/recipes/install-make.sh /setup/install-make.sh
|
||||
# %include taskcluster/docker/recipes/install-cmake.sh
|
||||
COPY topsrcdir/taskcluster/docker/recipes/install-cmake.sh /setup/install-cmake.sh
|
||||
|
||||
# %include taskcluster/docker/recipes/centos6-build-system-setup.sh
|
||||
COPY topsrcdir/taskcluster/docker/recipes/centos6-build-system-setup.sh /setup/system-setup.sh
|
||||
|
||||
# %include taskcluster/docker/recipes/centos-install-debug-symbols.sh
|
||||
COPY topsrcdir/taskcluster/docker/recipes/centos-install-debug-symbols.sh /setup/install-debug-symbols.sh
|
||||
|
||||
# TODO remove once base image doesn't install Mercurial
|
||||
RUN pip uninstall -y Mercurial
|
||||
# %include taskcluster/docker/recipes/debian-build-system-setup.sh
|
||||
COPY topsrcdir/taskcluster/docker/recipes/debian-build-system-setup.sh /setup/system-setup.sh
|
||||
|
||||
RUN bash /setup/system-setup.sh
|
||||
|
||||
# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
|
||||
# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb.
|
||||
# %include taskcluster/docker/recipes/xvfb.sh
|
||||
COPY topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh
|
||||
|
||||
# %include taskcluster/docker/recipes/run-task
|
||||
COPY topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task
|
||||
|
||||
# Add configuration
|
||||
# Add pip configuration, among other things.
|
||||
COPY dot-config /builds/worker/.config
|
||||
|
||||
# Generate machine uuid file
|
||||
RUN dbus-uuidgen --ensure=/var/lib/dbus/machine-id
|
||||
|
||||
# Stubbed out credentials; mozharness looks for this file an issues a WARNING
|
||||
# if it's not found, which causes the build to fail. Note that this needs to
|
||||
# be in the parent of the workspace directory and in the directory where
|
||||
# mozharness is run (not its --work-dir). See Bug 1169652.
|
||||
ADD oauth.txt /builds/worker/
|
||||
# %include taskcluster/docker/desktop-build/oauth.txt
|
||||
COPY topsrcdir/taskcluster/docker/desktop-build/oauth.txt /builds/worker/
|
||||
|
||||
# stubbed out buildprops, which keeps mozharness from choking
|
||||
# Add a stubbed out buildprops, which keeps mozharness from choking.
|
||||
# Note that this needs to be in the parent of the workspace directory and in
|
||||
# the directory where mozharness is run (not its --work-dir)
|
||||
ADD buildprops.json /builds/worker/
|
||||
|
||||
# END ../desktop-build/Dockerfile
|
||||
# the directory where mozharness is run (not its --work-dir).
|
||||
# %include taskcluster/docker/desktop-build/buildprops.json
|
||||
COPY topsrcdir/taskcluster/docker/desktop-build/buildprops.json /builds/worker/
|
||||
|
||||
# Reset user/workdir from parent image so we can install software.
|
||||
WORKDIR /
|
||||
USER root
|
||||
|
||||
# Update base.
|
||||
RUN rpm --rebuilddb && yum upgrade -y
|
||||
|
||||
# Install Sonatype Nexus. Cribbed directly from
|
||||
# https://github.com/sonatype/docker-nexus/blob/fffd2c61b2368292040910c055cf690c8e76a272/oss/Dockerfile.
|
||||
|
||||
# Install the screen package here to use with xvfb.
|
||||
# Move installation to base centos6-build image once Bug 1272629 is fixed
|
||||
# tzdata-java is symlinked from the JRE: see https://bugzilla.redhat.com/show_bug.cgi?id=1130800
|
||||
RUN rpm --rebuilddb && yum install -y \
|
||||
createrepo \
|
||||
curl \
|
||||
screen \
|
||||
sudo \
|
||||
tar \
|
||||
tzdata-java \
|
||||
unzip \
|
||||
wget \
|
||||
zip \
|
||||
&& yum clean all
|
||||
# Observe missing --no-install-recommends, in order to install glib2.0/gconf/etc.
|
||||
RUN apt-get install -yyq \
|
||||
libgconf2-4 \
|
||||
libgnome2-0 \
|
||||
openjdk-8-jdk-headless \
|
||||
&& \
|
||||
apt-get clean
|
||||
|
||||
ENV NEXUS_VERSION 2.12.0-01
|
||||
ENV NEXUS_SHA1SUM 1a9aaad8414baffe0a2fd46eed1f41b85f4049e6
|
||||
ENV NEXUS_VERSION=2.12.0-01 \
|
||||
NEXUS_SHA1SUM=1a9aaad8414baffe0a2fd46eed1f41b85f4049e6 \
|
||||
NEXUS_WORK=/builds/worker/workspace/nexus
|
||||
|
||||
RUN mkdir -p /opt/sonatype/nexus
|
||||
|
||||
@ -111,10 +165,7 @@ RUN tar zxf nexus-${NEXUS_VERSION}-bundle.tar.gz \
|
||||
&& rm -rf /tmp/nexus-${NEXUS_VERSION}-bundle.tar.gz
|
||||
|
||||
# So that we don't have to RUN_AS_USER=root.
|
||||
RUN chown -R worker:worker /opt/sonatype/nexus/
|
||||
RUN chown -R worker:worker /opt/sonatype
|
||||
|
||||
# Back to the centos6-build workdir, matching desktop-build.
|
||||
# Back to the workdir, matching desktop-build.
|
||||
WORKDIR /builds/worker
|
||||
|
||||
# Set a default command useful for debugging
|
||||
CMD ["/bin/bash", "--login"]
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"buildername": ""
|
||||
},
|
||||
"sourcestamp": {
|
||||
"changes": []
|
||||
},
|
||||
"comments": "TaskCluster Job"
|
||||
}
|
||||
@ -1,2 +0,0 @@
|
||||
taskcluster_clientId = None
|
||||
taskcluster_accessToken = None
|
||||
3
taskcluster/docker/android-build/sources.list
Normal file
3
taskcluster/docker/android-build/sources.list
Normal file
@ -0,0 +1,3 @@
|
||||
deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/20170830T000511Z/ stretch main
|
||||
deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/20170830T000511Z/ stretch-updates main
|
||||
deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/20170830T000511Z/ stretch/updates main
|
||||
11
taskcluster/docker/recipes/debian-build-system-setup.sh
Normal file
11
taskcluster/docker/recipes/debian-build-system-setup.sh
Normal file
@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
cd /setup
|
||||
|
||||
. /setup/common.sh
|
||||
. /setup/install-mercurial.sh
|
||||
|
||||
rm -rf /setup
|
||||
17
taskcluster/docker/recipes/install-mercurial.sh
Normal file → Executable file
17
taskcluster/docker/recipes/install-mercurial.sh
Normal file → Executable file
@ -27,6 +27,23 @@ if [ -f /etc/lsb-release ]; then
|
||||
|
||||
CERT_PATH=/etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
elif [ -f /etc/os-release ]; then
|
||||
. /etc/os-release
|
||||
|
||||
if [ "${ID}" = "debian" -a "${VERSION_ID}" = "9" ]; then
|
||||
if [ -f /usr/bin/pip2 ]; then
|
||||
PIP_PATH=/usr/bin/pip2
|
||||
else
|
||||
echo "We currently require Python 2.7 and /usr/bin/pip2 to run Mercurial"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Unsupported debian-like system with ID '${ID}' and VERSION_ID '${VERSION_ID}'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CERT_PATH=/etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
elif [ -f /etc/centos-release ]; then
|
||||
CENTOS_VERSION=`rpm -q --queryformat '%{VERSION}' centos-release`
|
||||
if [ "${CENTOS_VERSION}" = "6" ]; then
|
||||
|
||||
11
taskcluster/scripts/builder/build-android-dependencies/after.sh
Normal file → Executable file
11
taskcluster/scripts/builder/build-android-dependencies/after.sh
Normal file → Executable file
@ -19,17 +19,10 @@ tar cJf android-sdk-linux.tar.xz android-sdk-linux
|
||||
mkdir -p /builds/worker/private/android-sdk
|
||||
mv android-sdk-linux.tar.xz /builds/worker/private/android-sdk
|
||||
|
||||
cp -R $WORKSPACE/build/src/java_home java_home
|
||||
tar cJf java_home.tar.xz java_home
|
||||
|
||||
# We can't redistribute Java publicly.
|
||||
mkdir -p /builds/worker/private/java_home
|
||||
mv java_home.tar.xz /builds/worker/private/java_home
|
||||
|
||||
cp -R $WORKSPACE/nexus/storage/jcenter jcenter
|
||||
cp -R ${NEXUS_WORK}/storage/jcenter jcenter
|
||||
tar cJf jcenter.tar.xz jcenter
|
||||
|
||||
cp -R $WORKSPACE/nexus/storage/google google
|
||||
cp -R ${NEXUS_WORK}/storage/google google
|
||||
tar cJf google.tar.xz google
|
||||
|
||||
# The Gradle wrapper will have downloaded and verified the hash of exactly one
|
||||
|
||||
23
taskcluster/scripts/builder/build-android-dependencies/before.sh
Normal file → Executable file
23
taskcluster/scripts/builder/build-android-dependencies/before.sh
Normal file → Executable file
@ -8,21 +8,10 @@ echo "running as" $(id)
|
||||
|
||||
set -v
|
||||
|
||||
mkdir -p ${WORKSPACE}/nexus/conf
|
||||
cp /builds/worker/workspace/build/src/taskcluster/scripts/builder/build-android-dependencies/nexus.xml ${WORKSPACE}/nexus/conf/nexus.xml
|
||||
mkdir -p ${NEXUS_WORK}/conf
|
||||
cp /builds/worker/workspace/build/src/taskcluster/scripts/builder/build-android-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml
|
||||
|
||||
# Populate /builds/worker/workspace/build/src/java_home.
|
||||
. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
|
||||
|
||||
mv $WORKSPACE/java/usr/lib/jvm/java_home $WORKSPACE/build/src/java_home
|
||||
|
||||
export JAVA_HOME=$WORKSPACE/build/src/java_home
|
||||
export PATH=$PATH:$JAVA_HOME/bin
|
||||
|
||||
# Populate /builds/worker/.mozbuild/android-sdk-linux.
|
||||
python2.7 $WORKSPACE/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
|
||||
|
||||
RUN_AS_USER=worker NEXUS_WORK=$WORKSPACE/nexus /opt/sonatype/nexus/bin/nexus restart
|
||||
RUN_AS_USER=worker /opt/sonatype/nexus/bin/nexus restart
|
||||
|
||||
# Wait "a while" for Nexus to actually start. Don't fail if this fails.
|
||||
wget --quiet --retry-connrefused --waitretry=2 --tries=100 \
|
||||
@ -34,3 +23,9 @@ curl http://localhost:8081/nexus/service/local/status || true
|
||||
|
||||
# Verify Nexus has actually started. Fail if this fails.
|
||||
curl --fail --silent --location http://localhost:8081/nexus/service/local/status | grep '<state>STARTED</state>'
|
||||
|
||||
# It's helpful when debugging to see the repository configurations.
|
||||
curl http://localhost:8081/nexus/service/local/repositories || true
|
||||
|
||||
# Populate /builds/worker/.mozbuild/android-sdk-linux.
|
||||
python2.7 /builds/worker/workspace/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
|
||||
|
||||
Loading…
Reference in New Issue
Block a user