Bug 1212411: fork docker images for tests; r=ahal

This also refactors the image generation to not depend on puppetagain, and to
install the latest and greatest mercurial in a verified, repeatable way.

As with centos6-build-upd, the ubuntu1204-test-upd image exists to run
'apt-get upgrade' periodically without regenerating the huge parent image.

--HG--
extra : commitid : HxYSFfIaEc
extra : rebase_source : 28dae0a30be3d6fae2c44a43601bbd336d9bbfdd
This commit is contained in:
Dustin J. Mitchell 2015-10-09 10:49:03 -04:00
parent 5e6a32b16b
commit e277e99e09
22 changed files with 566 additions and 0 deletions

View File

@ -0,0 +1,42 @@
FROM taskcluster/ubuntu1204-test-upd:20151005212801
MAINTAINER Jonas Finnemann Jensen <jopsen@gmail.com>
# Add utilities and configuration
COPY dot-config /home/worker/.config
COPY dot-pulse /home/worker/.pulse
COPY bin /home/worker/bin
# TODO: remove these (they should be in-tree)
COPY mozharness_configs /home/worker/mozharness_configs
# TODO: remove this when buildbot is gone
COPY buildprops.json /home/worker/buildprops.json
COPY tc-vcs-config.yml /etc/taskcluster-vcs.yml
# TODO: remove
ADD https://raw.githubusercontent.com/taskcluster/buildbot-step/master/buildbot_step /home/worker/bin/buildbot_step
RUN chmod u+x /home/worker/bin/buildbot_step
# TODO: remove
ADD https://s3-us-west-2.amazonaws.com/test-caching/packages/linux64-stackwalk /usr/local/bin/linux64-minidump_stackwalk
RUN chmod u+x /usr/local/bin/linux64-minidump_stackwalk
RUN mkdir Documents; mkdir Pictures; mkdir Music; mkdir Videos; mkdir artifacts
# install a new enough npm, plus tc-vcs and tc-npm-cache
RUN npm install -g npm@^2.0.0
RUN npm install -g taskcluster-vcs@2.3.12
RUN npm install -g taskcluster-npm-cache@1.1.14
ENV PATH $PATH:/home/worker/bin
# Remove once running under 'worker' user. This is necessary for pulseaudio to start
# XXX: change this back to worker:worker once permissions issues are resolved
RUN chown -R root:root /home/worker
# TODO Re-enable worker when bug 1093833 lands
#USER worker
# clean up
RUN rm -Rf .cache && mkdir -p .cache
# Set a default command useful for debugging
CMD ["/bin/bash", "--login"]

View File

@ -0,0 +1 @@
taskcluster

View File

@ -0,0 +1 @@
0.4.0

View File

@ -0,0 +1,21 @@
#! /bin/bash -ex
# This file is intended to be used as the "entrypoint" or first command in the
# docker image. It uses "eval" so it takes place of the usual /bin/bash -c
# <stuff> conventions you need when running a docker image. Primarily it
# bootstraps any processes that need to run outside of mozharness (like Xvfb)
# XXX: Note that all tests could be faster at smaller bit depth
Xvfb :0 -nolisten tcp -screen 0 1600x1200x24 2>/dev/null &
export DISPLAY=:0
pulseaudio --start
# Download mozharness only when $MOZHARNESS_URL is defined
if [ ! -z "$MOZHARNESS_URL" ]; then
buildbot_step 'Download mozharness' wget -c $MOZHARNESS_URL && \
unzip -q mozharness.zip && \
rm mozharness.zip
fi
buildbot_step 'Running tests' $@

View File

@ -0,0 +1,13 @@
#!/bin/bash -vex
test $GAIA_BASE_REPOSITORY # ex: https://github.com/mozilla-b2g/gaia
test $GAIA_HEAD_REPOSITORY # ex: https://github.com/mozilla-b2g/gaia
test $GAIA_REF # ex: master
test $GAIA_REV # ex: master
tc-vcs \
checkout \
/home/worker/gaia \
$GAIA_BASE_REPOSITORY \
$GAIA_HEAD_REPOSITORY \
$GAIA_REV \
$GAIA_REF \

View File

@ -0,0 +1,8 @@
{
"properties": {
"buildername": ""
},
"sourcestamp": {
"changes": []
}
}

View File

@ -0,0 +1,4 @@
[global]
disable-pip-version-check = true
trusted-host = pypi.pub.build.mozilla.org

View File

@ -0,0 +1,15 @@
# This file is written by xdg-user-dirs-update
# If you want to change or add directories, just edit the line you're
# interested in. All local changes will be retained on the next run
# Format is XDG_xxx_DIR="$HOME/yyy", where yyy is a shell-escaped
# homedir-relative path, or XDG_xxx_DIR="/yyy", where /yyy is an
# absolute path. No other format is supported.
XDG_DESKTOP_DIR="$HOME/Desktop"
XDG_DOWNLOAD_DIR="$HOME/Downloads"
XDG_TEMPLATES_DIR="$HOME/Templates"
XDG_PUBLICSHARE_DIR="$HOME/Public"
XDG_DOCUMENTS_DIR="$HOME/Documents"
XDG_MUSIC_DIR="$HOME/Music"
XDG_PICTURES_DIR="$HOME/Pictures"
XDG_VIDEOS_DIR="$HOME/Videos"

View File

@ -0,0 +1 @@
en_US

View File

@ -0,0 +1,164 @@
#!/usr/bin/pulseaudio -nF
#
# This file is part of PulseAudio.
#
# PulseAudio is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# PulseAudio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PulseAudio; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
# This startup script is used only if PulseAudio is started per-user
# (i.e. not in system mode)
.nofail
### Load something into the sample cache
#load-sample-lazy x11-bell /usr/share/sounds/gtk-events/activate.wav
#load-sample-lazy pulse-hotplug /usr/share/sounds/startup3.wav
#load-sample-lazy pulse-coldplug /usr/share/sounds/startup3.wav
#load-sample-lazy pulse-access /usr/share/sounds/generic.wav
.fail
### Automatically restore the volume of streams and devices
load-module module-device-restore
load-module module-stream-restore
load-module module-card-restore
### Automatically augment property information from .desktop files
### stored in /usr/share/application
load-module module-augment-properties
### Load audio drivers statically
### (it's probably better to not load these drivers manually, but instead
### use module-udev-detect -- see below -- for doing this automatically)
#load-module module-alsa-sink
#load-module module-alsa-source device=hw:1,0
#load-module module-oss device="/dev/dsp" sink_name=output source_name=input
#load-module module-oss-mmap device="/dev/dsp" sink_name=output source_name=input
#load-module module-null-sink
#load-module module-pipe-sink
### Automatically load driver modules depending on the hardware available
.ifexists module-udev-detect.so
load-module module-udev-detect
.else
### Use the static hardware detection module (for systems that lack udev/hal support)
load-module module-detect
.endif
### Automatically connect sink and source if JACK server is present
.ifexists module-jackdbus-detect.so
.nofail
load-module module-jackdbus-detect
.fail
.endif
### Automatically load driver modules for Bluetooth hardware
# This module causes a pulseaudio startup failure on "gecko-tester"
#.ifexists module-bluetooth-discover.so
#load-module module-bluetooth-discover
#.endif
### Load several protocols
.ifexists module-esound-protocol-unix.so
load-module module-esound-protocol-unix
.endif
load-module module-native-protocol-unix
### Network access (may be configured with paprefs, so leave this commented
### here if you plan to use paprefs)
#load-module module-esound-protocol-tcp
#load-module module-native-protocol-tcp
#load-module module-zeroconf-publish
### Load the RTP receiver module (also configured via paprefs, see above)
#load-module module-rtp-recv
### Load the RTP sender module (also configured via paprefs, see above)
#load-module module-null-sink sink_name=rtp format=s16be channels=2 rate=44100 sink_properties="device.description='RTP Multicast Sink'"
#load-module module-rtp-send source=rtp.monitor
### Load additional modules from GConf settings. This can be configured with the paprefs tool.
### Please keep in mind that the modules configured by paprefs might conflict with manually
### loaded modules.
.ifexists module-gconf.so
.nofail
load-module module-gconf
.fail
.endif
### Automatically restore the default sink/source when changed by the user
### during runtime
### NOTE: This should be loaded as early as possible so that subsequent modules
### that look up the default sink/source get the right value
load-module module-default-device-restore
### Automatically move streams to the default sink if the sink they are
### connected to dies, similar for sources
load-module module-rescue-streams
### Make sure we always have a sink around, even if it is a null sink.
load-module module-always-sink
### Honour intended role device property
load-module module-intended-roles
### Automatically suspend sinks/sources that become idle for too long
load-module module-suspend-on-idle
### If autoexit on idle is enabled we want to make sure we only quit
### when no local session needs us anymore.
# This module causes a pulseaudio startup failure on "gecko-tester"
#.ifexists module-console-kit.so
#load-module module-console-kit
#.endif
### Enable positioned event sounds
load-module module-position-event-sounds
### Cork music streams when a phone stream is active
#load-module module-cork-music-on-phone
### Modules to allow autoloading of filters (such as echo cancellation)
### on demand. module-filter-heuristics tries to determine what filters
### make sense, and module-filter-apply does the heavy-lifting of
### loading modules and rerouting streams.
load-module module-filter-heuristics
load-module module-filter-apply
### Load DBus protocol
#.ifexists module-dbus-protocol.so
#load-module module-dbus-protocol
#.endif
# X11 modules should not be started from default.pa so that one daemon
# can be shared by multiple sessions.
### Load X11 bell module
#load-module module-x11-bell sample=bell-windowing-system
### Register ourselves in the X11 session manager
#load-module module-x11-xsmp
### Publish connection data in the X11 root window
#.ifexists module-x11-publish.so
#.nofail
#load-module module-x11-publish
#.fail
#.endif
load-module module-switch-on-port-available
### Make some devices default
#set-default-sink output
#set-default-source input

View File

@ -0,0 +1,4 @@
config = {
"exes": {},
"busybox_url": ""
}

View File

@ -0,0 +1,4 @@
config = {
"exes": {},
"xre_url": ""
}

View File

@ -0,0 +1,6 @@
config = {
# We bake this directly into the tester image now...
"download_minidump_stackwalk": False,
"minidump_stackwalk_path": "/usr/local/bin/linux64-minidump_stackwalk",
"exes": {}
}

View File

@ -0,0 +1,40 @@
# Default configuration used by the tc-vs tools these can be overridden by
# passing the config you wish to use over the command line...
git: git
hg: hg
repoCache:
# Repo url to clone when running repo init..
repoUrl: https://git.mozilla.org/external/google/gerrit/git-repo.git
# Version of repo to utilize...
repoRevision: master
# The root where all downloaded cache files are stored on the local machine...
cacheDir: '{{env.HOME}}/.tc-vcs-repo/'
# Name/prefixed used as part of the base url.
cacheName: sources/{{name}}.tar.gz
# Command used to upload the tarball
uploadTar: "curl --header 'Content-Type: application/x-tar' --header 'Content-Encoding: gzip' -X PUT --data-binary @'{{source}}' '{{url}}'"
# Large http get requests are often slower using nodes built in http layer so
# we utilize a subprocess which is responsible for fetching...
get: curl --connect-timeout 30 --speed-limit 500000 -L -o {{dest}} {{url}}
# Used to create clone tarball
compress: tar -czf {{dest}} {{source}}
# All cache urls use tar + gz this is the command used to extract those files
# downloaded by the "get" command.
extract: tar -x -z -C {{dest}} -f {{source}}
cloneCache:
# The root where all downloaded cache files are stored on the local machine...
cacheDir: '{{env.HOME}}/.tc-vcs/'
# Command used to upload the tarball
uploadTar: "curl --header 'Content-Type: application/x-tar' --header 'Content-Encoding: gzip' -X PUT --data-binary @'{{source}}' '{{url}}'"
# Large http get requests are often slower using nodes built in http layer so
# we utilize a subprocess which is responsible for fetching...
get: curl --connect-timeout 30 --speed-limit 500000 -L -o {{dest}} {{url}}
# Used to create clone tarball
compress: tar -czf {{dest}} {{source}}
# All cache urls use tar + gz this is the command used to extract those files
# downloaded by the "get" command.
extract: tar -x -z --strip-components 1 -C {{dest}} -f {{source}}
# Name/prefixed used as part of the base url.
cacheName: clones/{{name}}.tar.gz

View File

@ -0,0 +1,4 @@
GAIA_REV=tip
GAIA_REF=tip
GAIA_BASE_REPOSITORY=https://hg.mozilla.org/integration/gaia-central
GAIA_HEAD_REPOSITORY=https://hg.mozilla.org/integration/gaia-central

View File

@ -0,0 +1,10 @@
FROM taskcluster/ubuntu1204-test:0.1.0
MAINTAINER Dustin J. Mitchell <dustin@mozilla.com>
### update to latest from upstream repositories
# if this becomes a long list of packages, consider bumping the
# ubunt1204-test version
RUN apt-get update && apt-get upgrade -y --force-yes
# Set a default command useful for debugging
CMD ["/bin/bash", "--login"]

View File

@ -0,0 +1 @@
taskcluster

View File

@ -0,0 +1 @@
20151005212801

View File

@ -0,0 +1,20 @@
FROM ubuntu:12.04
MAINTAINER Jonas Finnemann Jensen <jopsen@gmail.com>
RUN useradd -d /home/worker -s /bin/bash -m worker
WORKDIR /home/worker
# install non-build specific dependencies in a single layer
ADD system-setup.sh /tmp/system-setup.sh
RUN bash /tmp/system-setup.sh
# Set variable normally configured at login, by the shells parent process, these
# are taken from GNU su manual
ENV HOME /home/worker
ENV SHELL /bin/bash
ENV USER worker
ENV LOGNAME worker
ENV HOSTNAME taskcluster-worker
# Set a default command useful for debugging
CMD ["/bin/bash", "--login"]

View File

@ -0,0 +1 @@
taskcluster

View File

@ -0,0 +1 @@
0.1.0

View File

@ -0,0 +1,204 @@
#!/usr/bin/env bash
set -ve
test `whoami` == 'root'
mkdir -p /setup
cd /setup
apt_packages=()
apt_packages+=('alsa-base')
apt_packages+=('alsa-utils')
apt_packages+=('autoconf2.13')
apt_packages+=('bluez-alsa')
apt_packages+=('bluez-alsa:i386')
apt_packages+=('bluez-cups')
apt_packages+=('bluez-gstreamer')
apt_packages+=('build-essential')
apt_packages+=('ca-certificates')
apt_packages+=('ccache')
apt_packages+=('curl')
apt_packages+=('fonts-kacst')
apt_packages+=('fonts-kacst-one')
apt_packages+=('fonts-liberation')
apt_packages+=('fonts-stix')
apt_packages+=('fonts-unfonts-core')
apt_packages+=('fonts-unfonts-extra')
apt_packages+=('fonts-vlgothic')
apt_packages+=('g++-multilib')
apt_packages+=('gcc-multilib')
apt_packages+=('gir1.2-gnomebluetooth-1.0')
apt_packages+=('git')
apt_packages+=('gstreamer0.10-alsa')
apt_packages+=('gstreamer0.10-ffmpeg')
apt_packages+=('gstreamer0.10-plugins-bad')
apt_packages+=('gstreamer0.10-plugins-base')
apt_packages+=('gstreamer0.10-plugins-good')
apt_packages+=('gstreamer0.10-plugins-ugly')
apt_packages+=('gstreamer0.10-tools')
apt_packages+=('libasound2-dev')
apt_packages+=('libasound2-plugins:i386')
apt_packages+=('libcanberra-pulse')
apt_packages+=('libcurl4-openssl-dev')
apt_packages+=('libdbus-1-dev')
apt_packages+=('libdbus-glib-1-dev')
apt_packages+=('libdrm-intel1:i386')
apt_packages+=('libdrm-nouveau1a:i386')
apt_packages+=('libdrm-radeon1:i386')
apt_packages+=('libdrm2:i386')
apt_packages+=('libexpat1:i386')
apt_packages+=('libgconf2-dev')
apt_packages+=('libgl1-mesa-dri')
apt_packages+=('libgl1-mesa-dri:i386')
apt_packages+=('libgl1-mesa-glx')
apt_packages+=('libgl1-mesa-glx:i386')
apt_packages+=('libglapi-mesa')
apt_packages+=('libglapi-mesa:i386')
apt_packages+=('libglu1-mesa')
apt_packages+=('libglu1-mesa:i386')
apt_packages+=('libgnome-bluetooth8')
apt_packages+=('libgstreamer-plugins-base0.10-dev')
apt_packages+=('libgstreamer0.10-dev')
apt_packages+=('libgtk2.0-dev')
apt_packages+=('libiw-dev')
apt_packages+=('libllvm2.9')
apt_packages+=('libllvm3.0:i386')
apt_packages+=('libncurses5:i386')
apt_packages+=('libnotify-dev')
apt_packages+=('libpulse-dev')
apt_packages+=('libpulse-mainloop-glib0:i386')
apt_packages+=('libpulsedsp:i386')
apt_packages+=('libsdl1.2debian:i386')
apt_packages+=('libsox-fmt-alsa')
apt_packages+=('libx11-xcb1:i386')
apt_packages+=('libxdamage1:i386')
apt_packages+=('libxfixes3:i386')
apt_packages+=('libxt-dev')
apt_packages+=('libxxf86vm1')
apt_packages+=('libxxf86vm1:i386')
apt_packages+=('llvm')
apt_packages+=('llvm-2.9')
apt_packages+=('llvm-2.9-dev')
apt_packages+=('llvm-2.9-runtime')
apt_packages+=('llvm-dev')
apt_packages+=('llvm-runtime')
apt_packages+=('mesa-common-dev')
apt_packages+=('nano')
apt_packages+=('pulseaudio')
apt_packages+=('pulseaudio-module-X11')
apt_packages+=('pulseaudio-module-bluetooth')
apt_packages+=('pulseaudio-module-gconf')
apt_packages+=('rlwrap')
apt_packages+=('screen')
apt_packages+=('software-properties-common')
apt_packages+=('sudo')
apt_packages+=('tar')
apt_packages+=('ttf-arphic-uming')
apt_packages+=('ttf-dejavu')
apt_packages+=('ttf-indic-fonts-core')
apt_packages+=('ttf-kannada-fonts')
apt_packages+=('ttf-oriya-fonts')
apt_packages+=('ttf-paktype')
apt_packages+=('ttf-punjabi-fonts')
apt_packages+=('ttf-sazanami-mincho')
apt_packages+=('unzip')
apt_packages+=('uuid')
apt_packages+=('vim')
apt_packages+=('wget')
apt_packages+=('xvfb')
apt_packages+=('yasm')
apt_packages+=('zip')
# Bug 1176031: need `xset` to disable screensavers
apt_packages+=('x11-xserver-utils')
# use Ubuntu's Python-2.7 (2.7.3 on Precise)
apt_packages+=('python-dev')
apt_packages+=('python-pip')
apt-get update
apt-get install -y --force-yes ${apt_packages[@]}
# set up tooltool (temporarily)
curl https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py > /setup/tooltool.py
tooltool_fetch() {
cat >manifest.tt
python /setup/tooltool.py fetch
rm manifest.tt
}
# install peep
tooltool_fetch <<'EOF'
[
{
"size": 26912,
"digest": "9d730ed7852d4d217aaddda959cd5f871ef1b26dd6c513a3780bbb04a5a93a49d6b78e95c2274451a1311c10cc0a72755b269dc9af62640474e6e73a1abec370",
"algorithm": "sha512",
"filename": "peep-2.4.1.tar.gz",
"unpack": false
}
]
EOF
pip install peep-2.4.1.tar.gz
# remaining Python utilities are installed with `peep` from upstream
# repositories; peep verifies file integrity for us
cat >requirements.txt <<'EOF'
# wheel
# sha256: 90pZQ6kAXB6Je8-H9-ivfgDAb6l3e5rWkfafn6VKh9g
# tarball:
# sha256: qryO8YzdvYoqnH-SvEPi_qVLEUczDWXbkg7zzpgS49w
virtualenv==13.1.2
# sha256: tQ9peOfTn-DLKY-j-j6c5B0jVnIdFV5SiPnFfl8T6ac
mercurial==3.5
EOF
peep install -r requirements.txt
# Install node
tooltool_fetch <<'EOF'
[
{
"size": 5676610,
"digest": "ce27b788dfd141a5ba7674332825fc136fe2c4f49a319dd19b3a87c8fffa7a97d86cbb8535661c9a68c9122719aa969fc6a8c886458a0df9fc822eec99ed130b",
"algorithm": "sha512",
"filename": "node-v0.10.36-linux-x64.tar.gz"
}
]
EOF
tar -C /usr/local -xz --strip-components 1 < node-*.tar.gz
node -v # verify
# Install a slightly newer version of libxcb
# See bug 975216 for the original build of these packages
# NOTE: if you're re-creating this, the tarball contains an `update.sh` which will rebuild the repository.
tooltool_fetch <<'EOF'
[
{
"size": 9708011,
"digest": "45005c7e1fdb4839f3bb96bfdb5e448672e7e40fa3cfc22ef646e2996a541f151e88c61b8a568cc00b7fcf5cb5e98e00c4e603acb0b73f85125582fa00aae76e",
"algorithm": "sha512",
"filename": "xcb-repo-1.8.1-2ubuntu2.1mozilla1.tgz"
}
]
EOF
tar -zxf xcb-repo-*.tgz
cp /etc/apt/sources.list sources.list.orig
echo "deb file://$PWD/xcb precise all" >> /etc/apt/sources.list
apt-get update
apt-get -q -y --force-yes install libxcb1 libxcb-render0 libxcb-shm0 libxcb-glx0 libxcb-glx0:i386
libxcb1_version=$(dpkg-query -s libxcb1 | grep ^Version | awk '{ print $2 }')
[ "$libxcb1_version" = "1.8.1-2ubuntu2.1mozilla1" ] || exit 1
cp sources.list.orig /etc/apt/sources.list
apt-get update
# clean up
cd /
rm -rf /setup ~/.ccache ~/.cache ~/.npm
apt-get clean
apt-get autoclean
rm $0