Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-11 04:15:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1383973 - Introduce single build script for run: using: mozharness. r=dustin

Browse Source 
The old process ran "before" and "after" steps as root.  The
mozharness script doesn't run as root, which required some small
changes to not run Sonatype Nexus as root.  Everything else is a
straight-forward move of the scripts out of the `android-gradle-build`
image and into `taskcluster/scripts`.

MozReview-Commit-ID: CqnNI33OKmb

--HG--
rename : taskcluster/docker/android-gradle-build/bin/after.sh => taskcluster/scripts/builder/build-android-dependencies/after.sh
rename : taskcluster/docker/android-gradle-build/bin/before.sh => taskcluster/scripts/builder/build-android-dependencies/before.sh
rename : taskcluster/docker/android-gradle-build/bin/repackage-jdk-centos.sh => taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
extra : rebase_source : f94e6b9b780f96038c60d3825039a0f94add0404
This commit is contained in:
Nick Alexander 2017-08-02 16:41:15 -07:00
parent 9d2b3f404c
commit 0100ff2b4a
10 changed files with 61 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
taskcluster/docker/android-gradle-build/Dockerfile
View File

@ -8,11 +8,6 @@ MAINTAINER Nick Alexander <nalexander@mozilla.com>
VOLUME /home/worker/workspace
VOLUME /home/worker/tooltool-cache
# Add build scripts; these are the entry points from the taskcluster worker, and
# operate on environment variables
COPY bin /home/worker/bin
RUN chmod +x /home/worker/bin/*
# %include python/mozbuild/mozbuild/action/tooltool.py
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /build/tooltool.py
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/tooltool.py
@ -115,7 +110,10 @@ RUN tar zxf nexus-${NEXUS_VERSION}-bundle.tar.gz \
&& rm -rf /tmp/nexus-${NEXUS_VERSION} \
&& rm -rf /tmp/nexus-${NEXUS_VERSION}-bundle.tar.gz
COPY nexus.xml /workspace/nexus/conf/nexus.xml
# So that we don't have to RUN_AS_USER=root.
RUN chown -R worker:worker /opt/sonatype/nexus/
COPY nexus.xml /home/worker/workspace/nexus/conf/nexus.xml
# Back to the centos6-build workdir, matching desktop-build.
WORKDIR /home/worker

16
taskcluster/docker/android-gradle-build/bin/after-checkout.sh
View File

@ -1,16 +0,0 @@
#!/bin/bash -vex
set -x -e
: WORKSPACE ${WORKSPACE:=/workspace}
set -v
# Populate /home/worker/workspace/build/src/java_home.
cp -R /workspace/java/usr/lib/jvm/java_home /home/worker/workspace/build/src
export JAVA_HOME=/home/worker/workspace/build/src/java_home
export PATH=$PATH:$JAVA_HOME/bin
# Populate /home/worker/.mozbuild/android-sdk-linux.
python2.7 /home/worker/workspace/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive

27
taskcluster/docker/android-gradle-build/bin/before.sh
View File

@ -1,27 +0,0 @@
#!/bin/bash -vex
set -x -e
: WORKSPACE ${WORKSPACE:=/workspace}
: GRADLE_VERSION ${GRADLE_VERSION:=2.7}
set -v
# Populate $WORKSPACE/java/urs/lib/jvm/java_home. $topsrcdir hasn't
# been checked out yet, so we can't put this directly into
# $topsrcdir/java_home.
. $HOME/bin/repackage-jdk-centos.sh
export JAVA_HOME=$WORKSPACE/java/usr/lib/jvm/java_home
export PATH=$PATH:$JAVA_HOME/bin
# Frowned upon, but simplest.
RUN_AS_USER=root NEXUS_WORK=${WORKSPACE}/nexus /opt/sonatype/nexus/bin/nexus restart
# Wait "a while" for Nexus to actually start. Don't fail if this fails.
wget --quiet --retry-connrefused --waitretry=2 --tries=100 \
http://localhost:8081/nexus/service/local/status || true
rm -rf status
# Verify Nexus has actually started. Fail if this fails.
curl --fail --silent --location http://localhost:8081/nexus/service/local/status | grep '<state>STARTED</state>'

31
taskcluster/docker/android-gradle-build/bin/build.sh
View File

@ -1,31 +0,0 @@
#! /bin/bash -vex
set -x -e -v
# TODO: when bug 1093833 is solved and tasks can run as non-root, reduce this
# to a simple fail-if-root check
if [ $(id -u) = 0 ]; then
# each of the caches we have mounted are owned by root, so update that ownership
# to 'worker'
for cache in /home/worker/.tc-vcs /home/worker/workspace /home/worker/tooltool-cache; do
if [ -d $cache ]; then
# -R probably isn't necessary forever, but it fixes some poisoned
# caches for now
chown -R worker:worker $cache
fi
done
# ..then drop privileges by re-running this script
exec su worker /home/worker/bin/build.sh
fi
####
# The default build works for any fx_desktop_build based mozharness job:
# via linux-build.sh
####
. $HOME/bin/checkout-sources.sh
. $HOME/bin/after-checkout.sh
. $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh

17
taskcluster/docker/android-gradle-build/bin/checkout-script.sh
View File

@ -1,17 +0,0 @@
#! /bin/bash -vex
set -x -e
# Inputs, with defaults
: GECKO_HEAD_REPOSITORY ${GECKO_HEAD_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
: GECKO_HEAD_REV ${GECKO_HEAD_REV:=default}
: SCRIPT_DOWNLOAD_PATH ${SCRIPT_DOWNLOAD_PATH:=$PWD}
: SCRIPT_PATH ${SCRIPT_PATH:?"script path must be set"}
set -v
# download script from the gecko repository
url=${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/${SCRIPT_PATH}
wget --directory-prefix=${SCRIPT_DOWNLOAD_PATH} $url
chmod +x `basename ${SCRIPT_PATH}`

55
taskcluster/docker/android-gradle-build/bin/checkout-sources.sh
View File

@ -1,55 +0,0 @@
#! /bin/bash -vex
set -x -e
# Inputs, with defaults
# mozharness builds use three repositories: gecko (source), mozharness (build
# scripts) and tools (miscellaneous) for each, specify *_REPOSITORY. If the
# revision is not in the standard repo for the codebase, specify *_BASE_REPO as
# the canonical repo to clone and *_HEAD_REPO as the repo containing the
# desired revision. For Mercurial clones, only *_HEAD_REV is required; for Git
# clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1
# as *_HEAD_REV.
: GECKO_REPOSITORY ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
: GECKO_BASE_REPOSITORY ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}}
: GECKO_HEAD_REPOSITORY ${GECKO_HEAD_REPOSITORY:=${GECKO_REPOSITORY}}
: GECKO_HEAD_REV ${GECKO_HEAD_REV:=default}
: GECKO_HEAD_REF ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}}
: TOOLS_REPOSITORY ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools}
: TOOLS_BASE_REPOSITORY ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}}
: TOOLS_HEAD_REPOSITORY ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}}
: TOOLS_HEAD_REV ${TOOLS_HEAD_REV:=default}
: TOOLS_HEAD_REF ${TOOLS_HEAD_REF:=${TOOLS_HEAD_REV}}
: TOOLS_DISABLE ${TOOLS_DISABLE:=false}
: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
set -v
# check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools)
if [ ! "$TOOLS_DISABLE" = true ]
then
tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_HEAD_REV $TOOLS_HEAD_REF
if [ ! -d build ]; then
mkdir -p build
ln -s $WORKSPACE/build/tools build/tools
fi
fi
# TODO - include tools repository in EXTRA_CHECKOUT_REPOSITORIES list
for extra_repo in $EXTRA_CHECKOUT_REPOSITORIES; do
BASE_REPO="${extra_repo}_BASE_REPOSITORY"
HEAD_REPO="${extra_repo}_HEAD_REPOSITORY"
HEAD_REV="${extra_repo}_HEAD_REV"
HEAD_REF="${extra_repo}_HEAD_REF"
DEST_DIR="${extra_repo}_DEST_DIR"
tc-vcs checkout ${!DEST_DIR} ${!BASE_REPO} ${!HEAD_REPO} ${!HEAD_REV} ${!HEAD_REF}
done
export GECKO_DIR=$WORKSPACE/build/src
tc-vcs checkout $GECKO_DIR $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF

15
taskcluster/scripts/builder/build-android-dependencies.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash -vex
set -x -e
echo "running as" $(id)
: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
set -v
. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/before.sh
. $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh
. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/after.sh

24
taskcluster/docker/android-gradle-build/bin/after.sh → taskcluster/scripts/builder/build-android-dependencies/after.sh
View File

@ -2,13 +2,15 @@
set -x -e
: WORKSPACE ${WORKSPACE:=/workspace}
echo "running as" $(id)
: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
: GRADLE_VERSION ${GRADLE_VERSION:=2.14.1}
set -v
# Package everything up.
pushd ${WORKSPACE}
pushd $WORKSPACE
cp -R /home/worker/.mozbuild/android-sdk-linux android-sdk-linux
tar cJf android-sdk-linux.tar.xz android-sdk-linux
@ -17,17 +19,17 @@ tar cJf android-sdk-linux.tar.xz android-sdk-linux
mkdir -p /home/worker/private/android-sdk
mv android-sdk-linux.tar.xz /home/worker/private/android-sdk
cp -R /home/worker/workspace/build/src/java_home java_home
cp -R $WORKSPACE/build/src/java_home java_home
tar cJf java_home.tar.xz java_home
# We can't redistribute Java publicly.
mkdir -p /home/worker/private/java_home
mv java_home.tar.xz /home/worker/private/java_home
cp -R /workspace/nexus/storage/jcenter jcenter
cp -R $WORKSPACE/nexus/storage/jcenter jcenter
tar cJf jcenter.tar.xz jcenter
cp -R /workspace/nexus/storage/google google
cp -R $WORKSPACE/nexus/storage/google google
tar cJf google.tar.xz google
# The Gradle wrapper will have downloaded and verified the hash of exactly one
@ -45,15 +47,3 @@ mv jcenter.tar.xz /home/worker/artifacts
mv google.tar.xz /home/worker/artifacts
mv gradle-dist.tar.xz /home/worker/artifacts
popd
# Bug 1245170: at some point in the future, we'll be able to upload
# things directly to tooltool.
# pushd /home/worker/artifacts
# /build/tooltool.py add --visibility=public jcenter.tar.xz
# /build/tooltool.py add --visibility=public google.tar.xz
# /build/tooltool.py add --visibility=public gradle-dist.tar.xz
# /build/tooltool.py add --visibility=internal android-sdk-linux.tar.xz
# /build/tooltool.py add --visibility=internal java_home.tar.xz
# /build/tooltool.py upload -v --url=http://relengapi/tooltool/ \
# --message="No message - Archives uploaded from taskcluster."
# popd

33
taskcluster/scripts/builder/build-android-dependencies/before.sh Normal file
View File

@ -0,0 +1,33 @@
#!/bin/bash -vex
set -x -e
echo "running as" $(id)
: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
set -v
# Populate /home/worker/workspace/build/src/java_home.
. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
mv $WORKSPACE/java/usr/lib/jvm/java_home $WORKSPACE/build/src/java_home
export JAVA_HOME=$WORKSPACE/build/src/java_home
export PATH=$PATH:$JAVA_HOME/bin
# Populate /home/worker/.mozbuild/android-sdk-linux.
python2.7 /home/worker/workspace/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
RUN_AS_USER=worker NEXUS_WORK=$WORKSPACE/nexus /opt/sonatype/nexus/bin/nexus restart
# Wait "a while" for Nexus to actually start. Don't fail if this fails.
wget --quiet --retry-connrefused --waitretry=2 --tries=100 \
http://localhost:8081/nexus/service/local/status || true
rm -rf status
# It's helpful when debugging to see the "latest state".
curl http://localhost:8081/nexus/service/local/status || true
# Verify Nexus has actually started. Fail if this fails.
curl --fail --silent --location http://localhost:8081/nexus/service/local/status | grep '<state>STARTED</state>'

4
taskcluster/docker/android-gradle-build/bin/repackage-jdk-centos.sh → taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
View File

@ -1,8 +1,8 @@
#! /bin/bash
#!/bin/bash -vex
set -e -x
: WORKSPACE ${WORKSPACE:=/workspace}
: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
set -v