Bug 1928084 - Add dbus-python and python-dbusmock wheels r=ahal,taskgraph-reviewers

Differential Revision: https://phabricator.services.mozilla.com/D227342

build/wheel_requirements.txt

@@ -0,0 +1,4 @@
+setuptools==75.0.0
+setuptools_scm==8.1.0
+meson==1.6.0
+ninja==1.11.1.1

taskcluster/docker/build-python-wheels/Dockerfile

@@ -0,0 +1,18 @@
+# This docker image is used to build python wheels.
+FROM          $DOCKER_IMAGE_PARENT
+MAINTAINER    Alexandre Lissy <alissy@mozilla.com>
+
+# We need to declare all potentially cache volumes as caches. Also,
+# making high I/O paths volumes increase I/O throughput because of
+# AUFS slowness.
+VOLUME  /builds/worker/.cache
+VOLUME  /builds/worker/checkouts
+VOLUME  /builds/worker/workspace
+
+RUN apt-get install build-essential \
+                    libdbus-1-dev \
+                    libglib2.0-dev \
+                    python3-dev \
+                    python3-pip
+
+CMD ["/bin/bash", "--login"]

taskcluster/docs/attributes.rst

@@ -541,3 +541,7 @@ Used by source tests to support chunking and specify a current chunk.
 total_chunks
 =============
 Used by source tests to support chunking and specify a total amount of chunks.
+
+toolchain-extract
+=================
+Control whether toolchain should be automatically extracted after download.

taskcluster/gecko_taskgraph/transforms/job/__init__.py

@@ -180,7 +180,7 @@ def get_attribute(dict, key, attributes, attribute_name):
     """Get `attribute_name` from the given `attributes` dict, and if there
     is a corresponding value, set `key` in `dict` to that value."""
     value = attributes.get(attribute_name)
-    if value:
+    if value is not None:
         dict[key] = value
 
 
@@ -223,6 +223,7 @@ def use_system_python(config, jobs):
 def use_fetches(config, jobs):
     artifact_names = {}
     extra_env = {}
+    should_extract = {}
     aliases = {}
     tasks = []
 
@@ -240,6 +241,9 @@ def use_fetches(config, jobs):
             artifact_names, task["label"], task["attributes"], f"{kind}-artifact"
         )
         get_attribute(extra_env, task["label"], task["attributes"], f"{kind}-env")
+        get_attribute(
+            should_extract, task["label"], task["attributes"], f"{kind}-extract"
+        )
         value = task["attributes"].get(f"{kind}-alias")
         if not value:
             value = []
@@ -289,7 +293,7 @@ def use_fetches(config, jobs):
                         {
                             "artifact": path,
                             "task": f"<{label}>",
-                            "extract": True,
+                            "extract": should_extract.get(label, True),
                         }
                     )
 

taskcluster/gecko_taskgraph/transforms/job/toolchain.py

@@ -5,7 +5,6 @@
 Support for running toolchain-building jobs via dedicated scripts
 """
 
-
 import os
 
 import taskgraph
@@ -63,6 +62,11 @@ toolchain_run_schema = Schema(
             "toolchain-env",
             description="Additional env variables to add to the worker when using this toolchain",
         ): {str: object},
+        Optional(
+            "toolchain-extract",
+            description="Whether the toolchain should be extracted after it is fetched "
+            + "(default: True)",
+        ): bool,
         # Base work directory used to set up the task.
         Optional("workdir"): str,
     }
@@ -157,6 +161,8 @@ def common_toolchain(config, job, taskdesc, is_docker):
         attributes["toolchain-alias"] = alias
     if "toolchain-env" in run:
         attributes["toolchain-env"] = run.pop("toolchain-env")
+    if "toolchain-extract" in run:
+        attributes["toolchain-extract"] = run.pop("toolchain-extract")
 
     # Allow the job to specify where artifacts come from, but add
     # public/build if it's not there already.

taskcluster/kinds/docker-image/kind.yml

@@ -73,6 +73,14 @@ tasks:
         parent: ubuntu2204-arm64-base
         definition: build-python
         arch: arm64
+    ubuntu1804-build-python-wheels:
+        symbol: I(ub18-build-python-wheels)
+        parent: ubuntu1804-base
+        definition: build-python-wheels
+    ubuntu2204-build-python-wheels:
+        symbol: I(ub22-build-python-wheels)
+        parent: ubuntu2204-base
+        definition: build-python-wheels
     # Neither the ubuntu1804-*raw nor the ubuntu1804-*packages images can have
     # packages dependencies.
     ubuntu2004-raw:

taskcluster/kinds/fetch/kind.yml

@@ -15,6 +15,7 @@ tasks-from:
     - browsertime.yml
     - chromium-fetch.yml
     - cft-chromedriver-fetch.yml
+    - python-wheels.yml
     - toolchain-clang-tidy-external.yml
     - toolchains.yml
     - translations-fetch.yml

taskcluster/kinds/fetch/python-wheels.yml

@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+dbus-python-1.2.18:
+    description: dbus-python-1.2.18 source
+    fetch:
+        type: static-url
+        url: https://files.pythonhosted.org/packages/b1/5c/ccfc167485806c1936f7d3ba97db6c448d0089c5746ba105b6eb22dba60e/dbus-python-1.2.18.tar.gz
+        sha256: 92bdd1e68b45596c833307a5ff4b217ee6929a1502f5341bae28fd120acf7260
+        size: 578204
+        artifact-name: dbus-python.tar.zst
+
+dbus-python-1.3.2:
+    description: dbus-python-1.3.2 source
+    fetch:
+        type: static-url
+        url: https://files.pythonhosted.org/packages/c1/d3/6be85a9c772d6ebba0cc3ab37390dd6620006dcced758667e0217fb13307/dbus-python-1.3.2.tar.gz
+        sha256: ad67819308618b5069537be237f8e68ca1c7fcc95ee4a121fe6845b1418248f8
+        size: 605495
+        artifact-name: dbus-python.tar.zst
+
+python-dbusmock-0.32.2:
+    description: python-dbusmock-0.32.2 source
+    fetch:
+        type: static-url
+        url: https://files.pythonhosted.org/packages/d2/b1/240ad35502c356f9f6e30ca246b66df6ff44b01a115920da27bf7a8d4ea7/python-dbusmock-0.32.2.tar.gz
+        sha256: 357c269d53a5bb3f87983bec7ff700a0c4d071481f7cf7221ec8376224fc4316
+        size: 104818
+        artifact-name: python-dbusmock.tar.zst

taskcluster/kinds/toolchain/kind.yml

@@ -47,6 +47,7 @@ tasks-from:
     - node.yml
     - pkgconf.yml
     - python.yml
+    - python-wheels.yml
     - rust.yml
     - rust-size.yml
     - sccache.yml

taskcluster/kinds/toolchain/python-wheels.yml

@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+task-defaults:
+    description: "python wheels build"
+    attributes:
+        local-toolchain: true
+    worker-type: b-linux-gcp
+    worker:
+        max-run-time: 3600
+    run:
+        toolchain-extract: false
+        script: build-python-wheel.sh
+        resources:
+            - 'build/wheel_requirements.txt'
+
+linux64-1804-dbus-python:
+    treeherder:
+        symbol: TL(py3.8-dbus-python)
+    fetches:
+        fetch:
+            - dbus-python-1.2.18
+        toolchain:
+            - linux64-python-3.8
+    run:
+        toolchain-artifact: public/build/dbus_python-1.2.18-cp38-cp38-linux_x86_64.whl
+    worker:
+        docker-image: {in-tree: ubuntu1804-build-python-wheels}
+
+linux64-2204-dbus-python:
+    treeherder:
+        symbol: TL(py3.10-dbus-python)
+    fetches:
+        fetch:
+            - dbus-python-1.3.2
+    run:
+        toolchain-artifact: public/build/dbus_python-1.3.2-cp310-cp310-linux_x86_64.whl
+    worker:
+        docker-image: {in-tree: ubuntu2204-build-python-wheels}
+
+linux-python-dbusmock:
+    treeherder:
+        symbol: TL(py-python-dbusmock)
+    fetches:
+        fetch:
+            - python-dbusmock-0.32.2
+        toolchain:
+            - linux64-python-3.8
+    run:
+        toolchain-artifact: public/build/python_dbusmock-0.32.2-py3-none-any.whl
+    worker:
+        docker-image: {in-tree: ubuntu1804-build-python-wheels}

taskcluster/scripts/misc/build-python-wheel.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This script builds the official interpreter for the python language,
+# while also packing in a few default extra packages.
+
+set -e
+set -x
+
+UPLOAD_DIR=$(pwd)/artifacts
+mkdir -p "${UPLOAD_DIR}"
+
+ls -hal "${MOZ_FETCHES_DIR}"
+WHEEL_DIR=$(find "$MOZ_FETCHES_DIR/" -maxdepth 1 -mindepth 1 -type d -not -name "python")
+
+export PATH="$MOZ_FETCHES_DIR/python/bin":/builds/worker/.local/bin:$PATH
+
+python3 -m pip install --upgrade pip==23.0
+python3 -m pip install -r "${GECKO_PATH}/build/wheel_requirements.txt"
+
+cd "$WHEEL_DIR"
+python3 setup.py bdist_wheel
+whl=$(ls dist/*.whl)
+
+cp "$whl" "$UPLOAD_DIR/"

testing/mozharness/mozharness/base/python.py

@@ -316,46 +316,55 @@ class VirtualenvMixin(object):
         fl_retry_loops = (fl_max_retry_minutes * 60) / fl_retry_sleep_seconds
         for link in c.get("find_links", []):
             parsed = urlparse.urlparse(link)
-            dns_result = None
-            get_result = None
-            retry_counter = 0
-            while retry_counter < fl_retry_loops and (
-                dns_result is None or get_result is None
-            ):
-                try:
-                    dns_result = socket.gethostbyname(parsed.hostname)
-                    get_result = urllib.request.urlopen(link, timeout=10).read()
-                    break
-                except socket.gaierror:
-                    retry_counter += 1
-                    self.warning(
-                        "find_links: dns check failed for %s, sleeping %ss and retrying..."
-                        % (parsed.hostname, fl_retry_sleep_seconds)
+            if parsed.scheme in ["http", "https"]:
+                dns_result = None
+                get_result = None
+                retry_counter = 0
+                while retry_counter < fl_retry_loops and (
+                    dns_result is None or get_result is None
+                ):
+                    try:
+                        dns_result = socket.gethostbyname(parsed.hostname)
+                        get_result = urllib.request.urlopen(link, timeout=10).read()
+                        break
+                    except socket.gaierror:
+                        retry_counter += 1
+                        self.warning(
+                            "find_links: dns check failed for %s, sleeping %ss and retrying..."
+                            % (parsed.hostname, fl_retry_sleep_seconds)
+                        )
+                        time.sleep(fl_retry_sleep_seconds)
+                    except (
+                        urllib.error.HTTPError,
+                        urllib.error.URLError,
+                        socket.timeout,
+                        http.client.RemoteDisconnected,
+                    ) as e:
+                        retry_counter += 1
+                        self.warning(
+                            "find_links: connection check failed for %s, sleeping %ss and retrying..."
+                            % (link, fl_retry_sleep_seconds)
+                        )
+                        self.warning("find_links: exception: %s" % e)
+                        time.sleep(fl_retry_sleep_seconds)
+                # now that the connectivity check is good, add the link
+                if dns_result and get_result:
+                    self.info(
+                        "find_links: connection checks passed for %s, adding." % link
                     )
-                    time.sleep(fl_retry_sleep_seconds)
-                except (
-                    urllib.error.HTTPError,
-                    urllib.error.URLError,
-                    socket.timeout,
-                    http.client.RemoteDisconnected,
-                ) as e:
-                    retry_counter += 1
+                    find_links_added += 1
+                    command.extend(["--find-links", link])
+                else:
                     self.warning(
-                        "find_links: connection check failed for %s, sleeping %ss and retrying..."
-                        % (link, fl_retry_sleep_seconds)
+                        "find_links: connection checks failed for %s"
+                        ", but max retries reached. continuing..." % link
                     )
-                    self.warning("find_links: exception: %s" % e)
-                    time.sleep(fl_retry_sleep_seconds)
-            # now that the connectivity check is good, add the link
-            if dns_result and get_result:
-                self.info("find_links: connection checks passed for %s, adding." % link)
+            elif len(parsed.path) > 0 and os.path.isdir(link):
+                self.info("find_links: dir exists %s, adding." % link)
                 find_links_added += 1
                 command.extend(["--find-links", link])
             else:
-                self.warning(
-                    "find_links: connection checks failed for %s"
-                    ", but max retries reached. continuing..." % link
-                )
+                self.warning("find_links: not a valid path nor URL %s" % link)
 
         # TODO: make this fatal if we always see failures after this
         if find_links_added == 0: