mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-08 19:04:45 +00:00
Bug 1839317 - Add Firefox Snap build of all branches using upstream toolings r=releng-reviewers,taskgraph-reviewers,jcristau
Differential Revision: https://phabricator.services.mozilla.com/D181502
This commit is contained in:
Alexandre Lissy
parent
cd28aa2e7e
commit
481771d6f0
@ -266,3 +266,17 @@ jobs:
|
||||
decision:
|
||||
symbol: I(decision)
|
||||
parent: ubuntu2004-base
|
||||
snap-build-core20:
|
||||
symbol: I(snap20-build)
|
||||
parent: ubuntu2004-base
|
||||
definition: snap-coreXX-build
|
||||
args:
|
||||
SNAP_BASE: core20
|
||||
SNAP_LIST: "gnome-3-38-2004-sdk"
|
||||
snap-build-core22:
|
||||
symbol: I(snap22-build)
|
||||
parent: ubuntu2204-base
|
||||
definition: snap-coreXX-build
|
||||
args:
|
||||
SNAP_BASE: core22
|
||||
SNAP_LIST: "gnome-42-2204 gtk-common-themes gnome-42-2204-sdk"
|
||||
|
||||
290
taskcluster/ci/snap-upstream-build/kind.yml
Normal file
290
taskcluster/ci/snap-upstream-build/kind.yml
Normal file
@ -0,0 +1,290 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: gecko_taskgraph.loader.transform:loader
|
||||
|
||||
kind-dependencies:
|
||||
- toolchain
|
||||
- fetch
|
||||
|
||||
transforms:
|
||||
- gecko_taskgraph.transforms.job:transforms
|
||||
- gecko_taskgraph.transforms.task:transforms
|
||||
|
||||
job-defaults:
|
||||
attributes:
|
||||
cron: true
|
||||
treeherder:
|
||||
kind: build
|
||||
tier: 2
|
||||
worker-type: b-linux-gcp
|
||||
worker:
|
||||
env:
|
||||
MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system
|
||||
|
||||
jobs:
|
||||
nightly:
|
||||
description: Build Firefox Nightly as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: nightly
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core22}
|
||||
max-run-time: 9000
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh nightly
|
||||
treeherder:
|
||||
platform: linux64/opt
|
||||
symbol: snap-nightly
|
||||
nightly-debug:
|
||||
description: Build Firefox Nightly (debug) as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: nightly-debug
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core22}
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh nightly 1
|
||||
treeherder:
|
||||
platform: linux64/debug
|
||||
symbol: snap-nightly
|
||||
beta:
|
||||
description: Build Firefox Beta as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: beta
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core22}
|
||||
max-run-time: 9000
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh beta
|
||||
treeherder:
|
||||
platform: linux64/opt
|
||||
symbol: snap-beta
|
||||
beta-debug:
|
||||
description: Build Firefox Beta (debug) as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: beta-debug
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core22}
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh beta 1
|
||||
treeherder:
|
||||
platform: linux64/debug
|
||||
symbol: snap-beta
|
||||
core22:
|
||||
description: Build Firefox Stable (Core22) as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: core22
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core22}
|
||||
max-run-time: 9000
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh core22
|
||||
treeherder:
|
||||
platform: linux64/opt
|
||||
symbol: snap-core22
|
||||
core22-debug:
|
||||
description: Build Firefox Stable (Core22) (debug) as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: core22-debug
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core22}
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh core22 1
|
||||
treeherder:
|
||||
platform: linux64/debug
|
||||
symbol: snap-core22
|
||||
stable:
|
||||
description: Build Firefox Stable as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: stable
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core20}
|
||||
max-run-time: 9000
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh stable
|
||||
treeherder:
|
||||
platform: linux64/opt
|
||||
symbol: snap-stable
|
||||
stable-debug:
|
||||
description: Build Firefox Stable (debug) as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: stable-debug
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core20}
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh stable 1
|
||||
treeherder:
|
||||
platform: linux64/debug
|
||||
symbol: snap-stable
|
||||
esr:
|
||||
description: Build Firefox ESR as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: esr
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core20}
|
||||
max-run-time: 9000
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh esr
|
||||
treeherder:
|
||||
platform: linux64/opt
|
||||
symbol: snap-esr
|
||||
esr-debug:
|
||||
description: Build Firefox ESR (debug) as a Snap using upstream definition
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: esr-debug
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core20}
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh esr 1
|
||||
treeherder:
|
||||
platform: linux64/debug
|
||||
symbol: snap-esr
|
||||
try:
|
||||
description: Build Firefox Nightly as a Snap using upstream definition and try repo
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: try
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core22}
|
||||
max-run-time: 9000
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh try
|
||||
treeherder:
|
||||
platform: linux64/opt
|
||||
symbol: snap-try
|
||||
try-debug:
|
||||
description: Build Firefox Nightly (debug) as a Snap using upstream definition and try repo
|
||||
attributes:
|
||||
build_platform: firefox-snap
|
||||
index:
|
||||
product: firefox
|
||||
job-name: try-debug
|
||||
type: generic
|
||||
worker:
|
||||
artifacts:
|
||||
- name: public/build
|
||||
type: directory
|
||||
path: /builds/worker/artifacts/
|
||||
docker-image: {in-tree: snap-build-core22}
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: run-task
|
||||
command: >-
|
||||
./run.sh try 1
|
||||
treeherder:
|
||||
platform: linux64/debug
|
||||
symbol: snap-try
|
||||
78
taskcluster/docker/snap-coreXX-build/Dockerfile
Normal file
78
taskcluster/docker/snap-coreXX-build/Dockerfile
Normal file
@ -0,0 +1,78 @@
|
||||
FROM $DOCKER_IMAGE_PARENT
|
||||
MAINTAINER Gabriele Svelto <gsvelto@mozilla.com>
|
||||
|
||||
RUN mkdir -p /builds
|
||||
RUN id worker || useradd -d /builds/worker -s /bin/bash -m worker
|
||||
WORKDIR /builds/worker
|
||||
|
||||
# We need to declare all potentially cache volumes as caches. Also,
|
||||
# making high I/O paths volumes increase I/O throughput because of
|
||||
# AUFS slowness.
|
||||
VOLUME /builds/worker/checkouts
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y \
|
||||
build-essential \
|
||||
curl \
|
||||
jq \
|
||||
patch \
|
||||
patchelf \
|
||||
python3-dev \
|
||||
python3-yaml \
|
||||
squashfs-tools \
|
||||
tar \
|
||||
unzip \
|
||||
uuid \
|
||||
wget \
|
||||
zip
|
||||
|
||||
ARG SNAP_BASE
|
||||
|
||||
COPY install-snap.sh /usr/local/bin/
|
||||
|
||||
RUN install-snap.sh core
|
||||
|
||||
RUN install-snap.sh core20
|
||||
|
||||
# Snapcraft snap depends on core20 for python3.8 even when we target core22
|
||||
RUN install-snap.sh snapcraft
|
||||
|
||||
RUN if [ "$SNAP_BASE" != "core20" ]; then install-snap.sh core22; fi
|
||||
|
||||
ARG SNAP_LIST
|
||||
|
||||
RUN for snap in $SNAP_LIST; do install-snap.sh "${snap}"; done
|
||||
|
||||
# Create a snapcraft runner
|
||||
RUN mkdir -p /snap/bin
|
||||
RUN echo "#!/bin/sh" > /snap/bin/snapcraft
|
||||
RUN snap_version="$(awk '/^version:/{print $2}' /snap/snapcraft/current/meta/snap.yaml)" && echo "export SNAP_VERSION=\"$snap_version\"\nexport PATH=\$SNAP/bin/:\$SNAP/libexec/snapcraft/:\$PATH" >> /snap/bin/snapcraft
|
||||
RUN echo 'exec "$SNAP/bin/snapcraft" "$@"' >> /snap/bin/snapcraft
|
||||
RUN chmod +x /snap/bin/snapcraft
|
||||
|
||||
# Generate locale
|
||||
RUN apt update && apt dist-upgrade --yes && apt install --yes sudo snapd locales && locale-gen en_US.UTF-8 && apt-get autoremove -y
|
||||
|
||||
# Set the proper environment
|
||||
ENV LANG="en_US.UTF-8"
|
||||
ENV LANGUAGE="en_US:en"
|
||||
ENV LC_ALL="en_US.UTF-8"
|
||||
ENV PATH="/snap/bin:$PATH"
|
||||
ENV SNAP="/snap/snapcraft/current"
|
||||
ENV SNAP_NAME="snapcraft"
|
||||
ENV SNAP_ARCH="amd64"
|
||||
|
||||
# Snapcraft pull will need to sudo apt-get update ...
|
||||
RUN echo "worker ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/worker && \
|
||||
chmod 0440 /etc/sudoers.d/worker
|
||||
|
||||
COPY run.sh /builds/worker/run.sh
|
||||
|
||||
COPY parse.py /builds/worker/parse.py
|
||||
|
||||
COPY patches /builds/worker/patches/
|
||||
|
||||
RUN chown -R worker:worker /builds/worker
|
||||
|
||||
# Set a default command useful for debugging
|
||||
CMD ["/bin/bash", "--login"]
|
||||
24
taskcluster/docker/snap-coreXX-build/install-snap.sh
Executable file
24
taskcluster/docker/snap-coreXX-build/install-snap.sh
Executable file
@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
SNAP_TO_INSTALL=$1
|
||||
|
||||
if [ -z "${SNAP_TO_INSTALL}" ]; then
|
||||
echo "Please give a snap name"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Grab the requested snap from the stable channel and unpack it in the proper
|
||||
# place (the 'Snap-CDN: none' header allows building in restricted network
|
||||
# environments such as Launchpad builders)
|
||||
|
||||
# shellcheck disable=SC2046
|
||||
curl -L \
|
||||
-H 'Snap-CDN: none' \
|
||||
$(curl -H 'X-Ubuntu-Series: 16' "https://api.snapcraft.io/api/v1/snaps/details/${SNAP_TO_INSTALL}?channel=stable" | jq '.download_url' -r) \
|
||||
--output "${SNAP_TO_INSTALL}.snap"
|
||||
|
||||
mkdir -p "/snap/${SNAP_TO_INSTALL}"
|
||||
|
||||
unsquashfs -d "/snap/${SNAP_TO_INSTALL}/current" "${SNAP_TO_INSTALL}.snap"
|
||||
39
taskcluster/docker/snap-coreXX-build/parse.py
Normal file
39
taskcluster/docker/snap-coreXX-build/parse.py
Normal file
@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def has_pkg_section(p, section):
|
||||
has_section = section in p.keys()
|
||||
if has_section:
|
||||
for pkg in p[section]:
|
||||
yield pkg
|
||||
|
||||
|
||||
def iter_pkgs(part, all_pkgs):
|
||||
for pkg in has_pkg_section(part, "build-packages"):
|
||||
if pkg not in all_pkgs:
|
||||
all_pkgs.append(pkg)
|
||||
for pkg in has_pkg_section(part, "stage-packages"):
|
||||
if pkg not in all_pkgs:
|
||||
all_pkgs.append(pkg)
|
||||
|
||||
|
||||
def parse(yaml_file):
|
||||
all_pkgs = []
|
||||
with open(yaml_file, "r") as inp:
|
||||
snap = yaml.safe_load(inp)
|
||||
parts = snap["parts"]
|
||||
for p in parts:
|
||||
iter_pkgs(parts[p], all_pkgs)
|
||||
return " ".join(all_pkgs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(parse(sys.argv[1]))
|
||||
@ -0,0 +1,19 @@
|
||||
diff --git a/snapcraft.yaml b/snapcraft.yaml
|
||||
index 78191e6..9add183 100644
|
||||
--- a/snapcraft.yaml
|
||||
+++ b/snapcraft.yaml
|
||||
@@ -23,14 +23,6 @@ assumes:
|
||||
- snapd2.54 # for mount-control
|
||||
compression: lzo
|
||||
|
||||
-package-repositories:
|
||||
- - type: apt
|
||||
- components: [main]
|
||||
- suites: [focal]
|
||||
- architectures: [arm64] # for gcc-11's libstdc++6
|
||||
- key-id: 60C317803A41BA51845E371A1E9377A2BA9EF27F
|
||||
- url: https://ppa.launchpadcontent.net/ubuntu-toolchain-r/test/ubuntu
|
||||
-
|
||||
apps:
|
||||
firefox:
|
||||
command: firefox.launcher
|
||||
@ -0,0 +1,19 @@
|
||||
diff --git a/snapcraft.yaml b/snapcraft.yaml
|
||||
index 93c0b60..319d564 100644
|
||||
--- a/snapcraft.yaml
|
||||
+++ b/snapcraft.yaml
|
||||
@@ -23,14 +23,6 @@ assumes:
|
||||
- snapd2.54 # for mount-control
|
||||
compression: lzo
|
||||
|
||||
-package-repositories:
|
||||
- - type: apt
|
||||
- components: [main]
|
||||
- suites: [focal]
|
||||
- architectures: [arm64] # for gcc-11's libstdc++6
|
||||
- key-id: 60C317803A41BA51845E371A1E9377A2BA9EF27F
|
||||
- url: https://ppa.launchpadcontent.net/ubuntu-toolchain-r/test/ubuntu
|
||||
-
|
||||
apps:
|
||||
firefox:
|
||||
command: firefox.launcher
|
||||
82
taskcluster/docker/snap-coreXX-build/run.sh
Executable file
82
taskcluster/docker/snap-coreXX-build/run.sh
Executable file
@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
mkdir -p /builds/worker/artifacts/
|
||||
mkdir -p /builds/worker/.local/state/snapcraft/
|
||||
ln -s /builds/worker/artifacts /builds/worker/.local/state/snapcraft/log
|
||||
|
||||
BRANCH=$1
|
||||
DEBUG=${2:-0}
|
||||
|
||||
export LC_ALL=C.UTF-8
|
||||
export LANG=C.UTF-8
|
||||
export SNAP_ARCH=amd64
|
||||
export SNAPCRAFT_BUILD_INFO=1
|
||||
|
||||
export PATH=$PATH:$HOME/.local/bin/
|
||||
unset MOZ_AUTOMATION
|
||||
|
||||
MOZCONFIG=mozconfig.in
|
||||
|
||||
# ESR currently still has a hard dependency against zstandard==0.17.0 so
|
||||
# install this specific version here
|
||||
if [ "${BRANCH}" = "esr" ]; then
|
||||
sudo apt-get remove -y python3-zstandard && sudo apt-get install -y python3-pip && sudo pip3 install --no-input zstandard==0.17.0
|
||||
MOZCONFIG=mozconfig
|
||||
fi
|
||||
|
||||
# Stable and beta runs out of file descriptors during link with gold
|
||||
ulimit -n 65536
|
||||
|
||||
TRY=0
|
||||
if [ "${BRANCH}" = "try" ]; then
|
||||
BRANCH=nightly
|
||||
TRY=1
|
||||
fi
|
||||
|
||||
git clone --single-branch --depth 1 --branch "${BRANCH}" https://github.com/canonical/firefox-snap/
|
||||
cd firefox-snap/
|
||||
|
||||
if [ "${TRY}" = "1" ]; then
|
||||
# Symlink so that we can directly re-use Gecko mercurial checkout
|
||||
ln -s /builds/worker/checkouts/gecko gecko
|
||||
fi
|
||||
|
||||
# Force an update to avoid the case of a stale docker image and repos updated
|
||||
# after
|
||||
sudo apt-get update
|
||||
|
||||
# shellcheck disable=SC2046
|
||||
sudo apt-get install -y $(python3 /builds/worker/parse.py snapcraft.yaml)
|
||||
|
||||
# CRAFT_PARTS_PACKAGE_REFRESH required to avoid snapcraft running apt-get update
|
||||
# especially for stage-packages
|
||||
if [ -d "/builds/worker/patches/${BRANCH}/" ]; then
|
||||
for p in /builds/worker/patches/"${BRANCH}"/*.patch; do
|
||||
patch -p1 < "$p"
|
||||
done;
|
||||
fi
|
||||
|
||||
if [ "${TRY}" = "1" ]; then
|
||||
# shellcheck disable=SC2016
|
||||
sed -ri 's|hg clone \$REPO -u \$REVISION|cp -r \$SNAPCRAFT_PROJECT_DIR/gecko/. |g' snapcraft.yaml
|
||||
else
|
||||
sed -ri "s|hg clone |hg clone --stream |g" snapcraft.yaml
|
||||
fi
|
||||
|
||||
if [ "${DEBUG}" = "1" ]; then
|
||||
echo "ac_add_options --enable-debug" >> ${MOZCONFIG}
|
||||
echo "MOZ_DEBUG=1" >> ${MOZCONFIG}
|
||||
|
||||
# No PGO on debug builds
|
||||
sed -ri 's/ac_add_options --enable-linker=gold//g' snapcraft.yaml
|
||||
sed -ri 's/ac_add_options --enable-lto=cross//g' snapcraft.yaml
|
||||
sed -ri 's/ac_add_options MOZ_PGO=1//g' snapcraft.yaml
|
||||
fi
|
||||
|
||||
SNAPCRAFT_BUILD_ENVIRONMENT_MEMORY=64G \
|
||||
SNAPCRAFT_BUILD_ENVIRONMENT_CPU=$(nproc) \
|
||||
CRAFT_PARTS_PACKAGE_REFRESH=0 \
|
||||
snapcraft --destructive-mode --verbose
|
||||
cp ./*.snap ./*.debug /builds/worker/artifacts/
|
||||
@ -794,3 +794,7 @@ Injects attribution information into en-US installers.
|
||||
attribution-l10n
|
||||
----------------
|
||||
Injects attribution information into localized installers.
|
||||
|
||||
snap-upstream-build
|
||||
-------------------
|
||||
Perform a Firefox Snap build using upstream tooling
|
||||
|
||||
Loading…
Reference in New Issue
Block a user