Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-23 21:01:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1921563 - Run snap build on Ubuntu 24.04 r=releng-reviewers,taskgraph-reviewers,jcristau

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D205705
This commit is contained in:
Alexandre Lissy 2024-10-01 13:58:59 +00:00
parent ae6d07beca
commit a54afb1db7
12 changed files with 118 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
taskcluster/config.yml
View File

@ -207,6 +207,7 @@ treeherder:
'Ub20': 'Packages for Ubuntu 20.04'
'Ub22': 'Packages for Ubuntu 22.04'
'Ub22a64': 'AArch64 packages for Ubuntu 22.04'
'Ub24': 'Packages for Ubuntu 24.04'
'I': 'Docker Image Builds'
'TA': 'Toolchain builds for Android'
'TL': 'Toolchain builds for Linux 64-bits'

3
taskcluster/docker/debian-base/Dockerfile
View File

@ -1,6 +1,9 @@
FROM $DOCKER_IMAGE_PARENT
MAINTAINER Mike Hommey <mhommey@mozilla.com>
# Ubuntu 24.04 now ships with user 1000 ...
RUN userdel --remove ubuntu || true
### Add worker user and setup its workspace.
RUN mkdir -p /builds && \
groupadd -g 1000 worker && \

5
taskcluster/docker/debian-raw/Dockerfile
View File

@ -19,6 +19,9 @@ COPY gpgvnoexpkeysig /usr/local/sbin
ARG DIST
ARG SNAPSHOT
ARG TASKCLUSTER_ROOT_URL
RUN if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then mv /etc/apt/sources.list.d/ubuntu.sources /etc/apt/; fi;
# Set apt sources list to a snapshot.
# Note: the use of gpgvnoexpkeysig is because the Debian Jessie GPG key expired.
RUN if [ -n "$DIST" ]; then for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST-security debian-debug_$DIST-debug debian-debug_$DIST-proposed-updates-debug debian-debug_$DIST-backports-debug; do \
@ -59,6 +62,8 @@ RUN if [ -n "$DIST" ]; then for s in debian_$DIST debian_$DIST-updates debian_$D
echo 'Pin-Priority: 1001'; \
) > /etc/apt/preferences.d/99taskcluster
RUN if [ -f /etc/apt/ubuntu.sources ]; then mv /etc/apt/ubuntu.sources /etc/apt/sources.list.d/; fi;
RUN apt-get update && \
if grep -q snapshot.debian.org /etc/apt/sources.list; then \
apt-get install python3-minimal libpython3-stdlib; \

14
taskcluster/docker/snap-coreXX-build/Dockerfile
View File

@ -10,12 +10,13 @@ WORKDIR /builds/worker
# AUFS slowness.
VOLUME /builds/worker/checkouts
ARG EXTRA_PACKAGES
RUN apt-get update && \
apt-get install -y \
build-essential \
curl \
libavcodec58 \
libavutil56 \
$EXTRA_PACKAGES \
jq \
patch \
patchelf \
@ -34,12 +35,9 @@ COPY install-snap.sh /usr/local/bin/
RUN install-snap.sh core
RUN install-snap.sh core20
# Snapcraft snap depends on core20 for python3.8 even when we target core22
RUN install-snap.sh snapcraft
RUN install-snap.sh core22
RUN install-snap.sh $SNAP_BASE
ARG SNAP_LIST
@ -63,6 +61,10 @@ ENV PATH="/snap/bin:$PATH"
ENV SNAP="/snap/snapcraft/current"
ENV SNAP_NAME="snapcraft"
ENV SNAP_ARCH="amd64"
ENV SNAP_INSTANCE_KEY=
ENV SNAP_INSTANCE_NAME=snapcraft
ENV SNAPCRAFT_BUILD_FOR=amd64
ENV SNAPCRAFT_PLATFORM=amd64
# Snapcraft pull will need to sudo apt-get update ...
RUN echo "worker ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/worker && \

2
taskcluster/docker/snap-coreXX-build/install-snap.sh
View File

@ -22,3 +22,5 @@ curl -L \
mkdir -p "/snap/${SNAP_TO_INSTALL}"
unsquashfs -d "/snap/${SNAP_TO_INSTALL}/current" "${SNAP_TO_INSTALL}.snap"
rm "${SNAP_TO_INSTALL}.snap"

18
taskcluster/docker/snap-coreXX-build/run.sh
View File

@ -26,7 +26,7 @@ USE_SNAP_FROM_STORE_OR_MC=${USE_SNAP_FROM_STORE_OR_MC:-0}
TRY=0
if [ "${BRANCH}" = "try" ]; then
BRANCH=nightly
SOURCE_BRANCH=nightly
TRY=1
fi
@ -90,8 +90,20 @@ if [ "${USE_SNAP_FROM_STORE_OR_MC}" = "0" ]; then
sed -ri 's/ac_add_options MOZ_PGO=1//g' snapcraft.yaml
fi
SNAPCRAFT_BUILD_ENVIRONMENT_MEMORY=64G \
SNAPCRAFT_BUILD_ENVIRONMENT_CPU=$(nproc) \
MAX_MEMORY_GB=$(free -g | awk '/Mem:/ { print $2 - 1 }')
# setting parallelism does not work with core24 ?
#
# SNAPCRAFT_BUILD_ENVIRONMENT_CPU=$(nproc) \
# SNAPCRAFT_PARALLEL_BUILD_COUNT=$(nproc) \
# CRAFT_PARALLEL_BUILD_COUNT=$(nproc) \
# Get the value and overwrite the snap's content.
MAX_CPUS=$(nproc)
sed -ri "s|\\\$CRAFT_PARALLEL_BUILD_COUNT|${MAX_CPUS}|g" snapcraft.yaml
grep "make -j" snapcraft.yaml
SNAPCRAFT_BUILD_ENVIRONMENT_MEMORY="${MAX_MEMORY_GB}G" \
CRAFT_PARTS_PACKAGE_REFRESH=0 \
snapcraft --destructive-mode --verbose
elif [ "${USE_SNAP_FROM_STORE_OR_MC}" = "store" ]; then

15
taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py
View File

@ -9,6 +9,7 @@ import io
import json
import os
import sys
import tempfile
import time
import traceback
@ -27,6 +28,10 @@ from selenium.webdriver.support.ui import WebDriverWait
class SnapTestsBase:
def __init__(self, exp):
snap_profile_path = tempfile.mkdtemp(
prefix="snap-tests",
dir=os.path.expanduser("~/snap/firefox/common/.mozilla/firefox/"),
)
driver_service = Service(
executable_path=r"/snap/firefox/current/usr/lib/firefox/geckodriver",
log_output=os.path.join(
@ -36,13 +41,15 @@ class SnapTestsBase:
options = Options()
if "TEST_GECKODRIVER_TRACE" in os.environ.keys():
options.log.level = "trace"
options.binary_location = r"/snap/firefox/current/usr/lib/firefox/firefox"
options.binary_location = r"/snap/bin/firefox"
if not "TEST_NO_HEADLESS" in os.environ.keys():
options.add_argument("--headless")
if "MOZ_AUTOMATION" in os.environ.keys():
os.environ["MOZ_LOG_FILE"] = os.path.join(
os.environ.get("ARTIFACT_DIR"), "gecko.log"
)
options.add_argument("-profile")
options.add_argument(snap_profile_path)
self._driver = webdriver.Firefox(service=driver_service, options=options)
self._logger = structuredlog.StructuredLogger(self.__class__.__name__)
@ -105,10 +112,12 @@ class SnapTestsBase:
test_status = "TIMEOUT"
test_message = repr(ex)
self.save_screenshot("screenshot_{}.png".format(test_status.lower()))
self.save_screenshot(
"screenshot_{}_{}.png".format(m.lower(), test_status.lower())
)
self._driver.switch_to.parent_frame()
self.save_screenshot(
"screenshot_{}_parent.png".format(test_status.lower())
"screenshot_{}_{}_parent.png".format(m.lower(), test_status.lower())
)
self._logger.test_end(m, status=test_status, message=test_message)
traceback.print_exc()

8
taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py
View File

@ -792,7 +792,9 @@ class QATests(SnapTestsBase):
)
if not os.path.isabs(previous_folder):
previous_folder = os.path.join(os.environ.get("HOME", ""), previous_folder)
with tempfile.TemporaryDirectory() as tmpdir:
with tempfile.TemporaryDirectory(
dir=os.environ.get("HOME"), prefix="snap-test-download"
) as tmpdir:
assert os.path.isdir(tmpdir), "tmpdir download should exists"
download_1 = os.path.abspath(os.path.join(previous_folder, download_name))
@ -820,7 +822,9 @@ class QATests(SnapTestsBase):
download_site = self.open_lafibre()
extra_small = self.get_lafibre_1M()
with tempfile.TemporaryDirectory() as tmpdir:
with tempfile.TemporaryDirectory(
dir=os.environ.get("HOME"), prefix="snap-test-download-rm"
) as tmpdir:
self.change_download_folder(None, tmpdir)
self._driver.switch_to.window(download_site)

1
taskcluster/gecko_taskgraph/transforms/job/distro_package.py
View File

@ -236,5 +236,6 @@ def docker_worker_ubuntu_package(config, job, taskdesc):
"bionic": 1804,
"focal": 2004,
"jammy": 2204,
"noble": 2404,
}[run["dist"]]
common_package(config, job, taskdesc, "ubuntu", version)

26
taskcluster/kinds/docker-image/kind.yml
View File

@ -96,6 +96,11 @@ tasks:
definition: debian-packages
parent: ubuntu2204-arm64-raw
arch: arm64
ubuntu2404-raw:
symbol: I(ub24-raw)
definition: debian-raw
args:
BASE_IMAGE: ubuntu:noble
ubuntu2004-packages:
symbol: I(ub20-pkg)
definition: debian-packages
@ -104,6 +109,10 @@ tasks:
symbol: I(ub22-pkg)
definition: debian-packages
parent: ubuntu2204-raw
ubuntu2404-packages:
symbol: I(ub24-pkg)
definition: debian-packages
parent: ubuntu2404-raw
ubuntu2004-base:
symbol: I(ub20-base)
definition: debian-base
@ -126,6 +135,13 @@ tasks:
- ub22-arm64-mercurial
- ub22-arm64-python-zstandard
arch: arm64
ubuntu2404-base:
symbol: I(ub24-base)
definition: debian-base
parent: ubuntu2404-raw
packages:
- ub24-mercurial
- ub24-python-zstandard
# Neither the debian8-*raw nor the debian8-*packages images can have
# packages dependencies.
debian8-raw:
@ -273,7 +289,15 @@ tasks:
definition: snap-coreXX-build
args:
SNAP_BASE: core22
SNAP_LIST: "gnome-42-2204 gtk-common-themes gnome-42-2204-sdk"
SNAP_LIST: "core20 gnome-42-2204 gtk-common-themes gnome-42-2204-sdk"
EXTRA_PACKAGES: "libavcodec58 libavutil56"
snap-build-core24:
symbol: I(snap24-build)
parent: ubuntu2404-base
definition: snap-coreXX-build
args:
SNAP_BASE: core24
SNAP_LIST: "core22 gnome-46-2404 gtk-common-themes gnome-46-2404-sdk mesa-2404"
android-components:
symbol: I(android-components)
parent: debian12-base

27
taskcluster/kinds/packages/ubuntu.yml
View File

@ -198,3 +198,30 @@ ub22-arm64-mercurial:
sed -i -e "s/__VERSION__/$(awk -F\" '$2 {print $2}' mercurial/__version__.py)-1.ub22moz1/" \
-e "s/__DATE__/$(date --rfc-2822)/" \
-e "s/__CODENAME__/jammy/" debian/changelog
ub24-python-zstandard:
description: "python-zstandard for Ubuntu noble"
treeherder:
symbol: Ub24(python-zstandard)
run:
tarball:
url: https://github.com/indygreg/python-zstandard/releases/download/0.22.0/zstandard-0.22.0.tar.gz
sha256: 8226a33c542bcb54cd6bd0a366067b610b41713b64c9abec1bc4533d69f51e70
name: python-zstandard
dist: noble
pre-build-command: debchange -v 0.22.0-1.ub24moz --distribution noble "Mozilla backport for noble"
ub24-mercurial:
description: "Modern Mercurial for Ubuntu noble"
treeherder:
symbol: Ub24(hg)
run:
tarball:
url: https://www.mercurial-scm.org/release/mercurial-6.8.1.tar.gz
sha256: 030e8a7a6d590e4eaeb403ee25675615cd80d236f3ab8a0b56dcc84181158b05
dist: noble
pre-build-command: >-
cp -r contrib/packaging/debian debian &&
sed -i -e "s/__VERSION__/$(awk -F\" '$2 {print $2}' mercurial/__version__.py)-1.ub24moz1/" \
-e "s/__DATE__/$(date --rfc-2822)/" \
-e "s/__CODENAME__/noble/" debian/changelog

15
taskcluster/kinds/snap-upstream-build/kind.yml
View File

@ -19,7 +19,7 @@ task-defaults:
kind: build
tier: 2
symbol: B
worker-type: b-linux-gcp
worker-type: b-linux-xlarge-gcp
run:
checkout: true
worker:
@ -27,7 +27,6 @@ task-defaults:
- name: public/build
type: directory
path: /builds/worker/artifacts/
docker-image: {in-tree: snap-build-core22}
max-run-time: 900
env:
UPLOAD_DIR: artifacts
@ -47,6 +46,7 @@ tasks:
job-name: amd64-nightly
type: generic
worker:
docker-image: {in-tree: snap-build-core24}
max-run-time: 9000
run:
using: run-task
@ -65,6 +65,7 @@ tasks:
job-name: amd64-nightly-debug
type: generic
worker:
docker-image: {in-tree: snap-build-core24}
max-run-time: 7200
run:
using: run-task
@ -83,6 +84,7 @@ tasks:
job-name: amd64-beta
type: generic
worker:
docker-image: {in-tree: snap-build-core22}
max-run-time: 9000
run:
using: run-task
@ -101,6 +103,7 @@ tasks:
job-name: amd64-beta-debug
type: generic
worker:
docker-image: {in-tree: snap-build-core22}
max-run-time: 7200
run:
using: run-task
@ -119,6 +122,7 @@ tasks:
job-name: amd64-stable
type: generic
worker:
docker-image: {in-tree: snap-build-core22}
max-run-time: 9000
run:
using: run-task
@ -137,6 +141,7 @@ tasks:
job-name: amd64-stable-debug
type: generic
worker:
docker-image: {in-tree: snap-build-core22}
max-run-time: 7200
run:
using: run-task
@ -155,6 +160,7 @@ tasks:
job-name: amd64-esr
type: generic
worker:
docker-image: {in-tree: snap-build-core22}
max-run-time: 9000
run:
using: run-task
@ -173,6 +179,7 @@ tasks:
job-name: amd64-esr-debug
type: generic
worker:
docker-image: {in-tree: snap-build-core22}
max-run-time: 7200
run:
using: run-task
@ -191,6 +198,7 @@ tasks:
job-name: amd64-esr128
type: generic
worker:
docker-image: {in-tree: snap-build-core22}
max-run-time: 9000
run:
using: run-task
@ -209,6 +217,7 @@ tasks:
job-name: amd64-esr128-debug
type: generic
worker:
docker-image: {in-tree: snap-build-core22}
max-run-time: 7200
run:
using: run-task
@ -227,6 +236,7 @@ tasks:
job-name: amd64-try
type: generic
worker:
docker-image: {in-tree: snap-build-core24}
max-run-time: 9000
run:
using: run-task
@ -245,6 +255,7 @@ tasks:
job-name: amd64-try-debug
type: generic
worker:
docker-image: {in-tree: snap-build-core24}
max-run-time: 7200
run:
using: run-task