Bug 1745246 - Allow specifying multiple maven artifacts from build. r=calu,ahal

This patch introduces a new field maven-artifacts which allows specifying a
list of maven artifacts that will be generated by a build and published to our
maven repository.

This allows us to split up GeckoView into separate maven packages.

Differential Revision: https://phabricator.services.mozilla.com/D133790

mobile/android/geckoview/build.gradle

@@ -513,7 +513,7 @@ publishing {
     }
     repositories {
         maven {
-            url = "${project.buildDir}/maven"
+            url = "${topobjdir}/gradle/maven"
         }
     }
 }

substitute-local-geckoview.gradle

@@ -104,7 +104,7 @@ def (mozconfig, topobjdir) = rootProject.ext.get(LOAD_MOZCONFIG_CACHE)
 repositories {
     maven {
         name "Local GeckoView Maven repository"
-        url "${topobjdir}/gradle/build/mobile/android/geckoview/maven"
+        url "${topobjdir}/gradle/maven"
     }
 }
 

taskcluster/ci/build-fat-aar/kind.yml

@@ -19,6 +19,8 @@ transforms:
 job-defaults:
     attributes:
         artifact_map: taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml
+        maven_packages:
+            - geckoview
     index:
         product: mobile
     worker-type: b-linux
@@ -32,7 +34,7 @@ job-defaults:
             MOZ_ANDROID_FAT_AAR_ARCHITECTURES: "armeabi-v7a,arm64-v8a,x86,x86_64"
         artifacts:
             - name: public/build/maven
-              path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/maven/
+              path: /builds/worker/workspace/obj-build/gradle/maven/
               type: directory
             - name: public/build
               path: /builds/worker/artifacts/

taskcluster/ci/build/android-asan.yml

@@ -5,6 +5,8 @@
 job-defaults:
     attributes:
         artifact_map: taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml
+        maven_packages:
+            - geckoview
     index:
         product: mobile
     worker:
@@ -14,7 +16,7 @@ job-defaults:
             GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline"
         artifacts:
             - name: public/build/maven
-              path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/maven/
+              path: /builds/worker/workspace/obj-build/gradle/maven/
               type: directory
             - name: public/build/geckoview-androidTest.apk
               path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk

taskcluster/ci/build/android.yml

@@ -5,6 +5,8 @@
 job-defaults:
     attributes:
         artifact_map: taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml
+        maven_packages:
+            - geckoview
     index:
         product: mobile
     worker:
@@ -14,7 +16,7 @@ job-defaults:
             GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline"
         artifacts:
             - name: public/build/maven
-              path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/maven/
+              path: /builds/worker/workspace/obj-build/gradle/maven/
               type: directory
             - name: public/build/geckoview-androidTest.apk
               path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk

taskcluster/docs/attributes.rst

@@ -433,6 +433,10 @@ This deliberately excludes builds that are implemented using the artifact build
 machinery, but are not primarily intended to short-circuit build time. In
 particular the Windows aarch64 builds are not marked this way.
 
+maven_packages
+===============
+List of maven packages produced by the build.
+
 supports-artifact-builds
 ========================
 

taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py

@@ -18,12 +18,11 @@ from gecko_taskgraph.util.attributes import (
     release_level,
 )
 from gecko_taskgraph.util.declarative_artifacts import (
-    get_geckoview_template_vars,
+    get_geckoview_artifact_map,
     get_geckoview_upstream_artifacts,
     get_geckoview_artifact_id,
 )
 from gecko_taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
-from gecko_taskgraph.util.scriptworker import generate_beetmover_artifact_map
 from gecko_taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
@@ -71,6 +70,17 @@ def resolve_keys(config, jobs):
         yield job
 
 
+@transforms.add
+def split_maven_packages(config, jobs):
+    for job in jobs:
+        dep_job = job["primary-dependency"]
+        attributes = copy_attributes_from_dependent_job(dep_job)
+        for package in attributes["maven_packages"]:
+            package_job = deepcopy(job)
+            package_job["maven-package"] = package
+            yield package_job
+
+
 @transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
@@ -79,7 +89,6 @@ def make_task_description(config, jobs):
         attributes.update(job.get("attributes", {}))
 
         treeherder = job.get("treeherder", {})
-        treeherder.setdefault("symbol", "BM-gv")
         dep_th_platform = (
             dep_job.task.get("extra", {})
             .get("treeherder", {})
@@ -89,6 +98,8 @@ def make_task_description(config, jobs):
         treeherder.setdefault("platform", f"{dep_th_platform}/opt")
         treeherder.setdefault("tier", 2)
         treeherder.setdefault("kind", "build")
+        package = job["maven-package"]
+        treeherder.setdefault("symbol", f"BM-{package}")
         label = job["label"]
         description = (
             "Beetmover submission for geckoview"
@@ -107,7 +118,7 @@ def make_task_description(config, jobs):
         attributes["run_on_hg_branches"] = job["run-on-hg-branches"]
 
         task = {
-            "label": label,
+            "label": f"{package}-{label}",
             "description": description,
             "worker-type": "beetmover",
             "scopes": [
@@ -119,6 +130,7 @@ def make_task_description(config, jobs):
             "run-on-projects": job["run-on-projects"],
             "treeherder": treeherder,
             "shipping-phase": job["shipping-phase"],
+            "maven-package": package,
         }
 
         yield task
@@ -128,19 +140,14 @@ def make_task_description(config, jobs):
 def make_task_worker(config, jobs):
     for job in jobs:
         job["worker"] = {
-            "artifact-map": generate_beetmover_artifact_map(
-                config,
-                job,
-                **get_geckoview_template_vars(
-                    config,
-                    job["attributes"]["build_platform"],
-                    job["attributes"].get("update-channel"),
-                ),
-            ),
+            "artifact-map": get_geckoview_artifact_map(config, job),
             "implementation": "beetmover-maven",
             "release-properties": craft_release_properties(config, job),
-            "upstream-artifacts": get_geckoview_upstream_artifacts(config, job),
+            "upstream-artifacts": get_geckoview_upstream_artifacts(
+                config, job, job["maven-package"]
+            ),
         }
+        del job["maven-package"]
 
         yield job
 
@@ -151,6 +158,7 @@ def craft_release_properties(config, job):
     release_properties["artifact-id"] = get_geckoview_artifact_id(
         config,
         job["attributes"]["build_platform"],
+        job["maven-package"],
         job["attributes"].get("update-channel"),
     )
     release_properties["app-name"] = "geckoview"

taskcluster/gecko_taskgraph/transforms/build_fat_aar.py

@@ -62,9 +62,13 @@ def set_fetches_and_locations(config, jobs):
 
 
 def _get_aar_location(config, job, platform):
-    artifacts_locations = get_geckoview_upstream_artifacts(
-        config, job, platform=platform
-    )
+    artifacts_locations = []
+
+    for package in job["attributes"]["maven_packages"]:
+        artifacts_locations += get_geckoview_upstream_artifacts(
+            config, job, package, platform=platform
+        )
+
     aar_locations = [
         path for path in artifacts_locations[0]["paths"] if path.endswith(".aar")
     ]

taskcluster/gecko_taskgraph/util/attributes.py

@@ -61,6 +61,7 @@ _COPYABLE_ATTRIBUTES = (
     "l10n_chunk",
     "locale",
     "mar-channel-id",
+    "maven_packages",
     "nightly",
     "required_signoffs",
     "shippable",

taskcluster/gecko_taskgraph/util/declarative_artifacts.py

@@ -5,29 +5,45 @@
 
 import re
 
-from gecko_taskgraph.util.scriptworker import generate_beetmover_upstream_artifacts
+from gecko_taskgraph.util.scriptworker import (
+    generate_beetmover_upstream_artifacts,
+    generate_beetmover_artifact_map,
+)
 
 
 _ARTIFACT_ID_PER_PLATFORM = {
-    "android-aarch64-opt": "geckoview-default-arm64-v8a",
-    "android-arm-opt": "geckoview-default-armeabi-v7a",
-    "android-x86-opt": "geckoview-default-x86",
-    "android-x86_64-opt": "geckoview-default-x86_64",
-    "android-geckoview-fat-aar-opt": "geckoview-default",
-    "android-aarch64-shippable": "geckoview{update_channel}-omni-arm64-v8a",
-    "android-aarch64-shippable-lite": "geckoview{update_channel}-arm64-v8a",
-    "android-arm-shippable": "geckoview{update_channel}-omni-armeabi-v7a",
-    "android-arm-shippable-lite": "geckoview{update_channel}-armeabi-v7a",
-    "android-x86-shippable": "geckoview{update_channel}-omni-x86",
-    "android-x86-shippable-lite": "geckoview{update_channel}-x86",
-    "android-x86_64-shippable": "geckoview{update_channel}-omni-x86_64",
-    "android-x86_64-shippable-lite": "geckoview{update_channel}-x86_64",
-    "android-geckoview-fat-aar-shippable": "geckoview{update_channel}-omni",
-    "android-geckoview-fat-aar-shippable-lite": "geckoview{update_channel}",
+    "android-aarch64-opt": "{package}-default-arm64-v8a",
+    "android-arm-opt": "{package}-default-armeabi-v7a",
+    "android-x86-opt": "{package}-default-x86",
+    "android-x86_64-opt": "{package}-default-x86_64",
+    "android-geckoview-fat-aar-opt": "{package}-default",
+    "android-aarch64-shippable": "{package}{update_channel}-omni-arm64-v8a",
+    "android-aarch64-shippable-lite": "{package}{update_channel}-arm64-v8a",
+    "android-arm-shippable": "{package}{update_channel}-omni-armeabi-v7a",
+    "android-arm-shippable-lite": "{package}{update_channel}-armeabi-v7a",
+    "android-x86-shippable": "{package}{update_channel}-omni-x86",
+    "android-x86-shippable-lite": "{package}{update_channel}-x86",
+    "android-x86_64-shippable": "{package}{update_channel}-omni-x86_64",
+    "android-x86_64-shippable-lite": "{package}{update_channel}-x86_64",
+    "android-geckoview-fat-aar-shippable": "{package}{update_channel}-omni",
+    "android-geckoview-fat-aar-shippable-lite": "{package}{update_channel}",
 }
 
 
-def get_geckoview_upstream_artifacts(config, job, platform=""):
+def get_geckoview_artifact_map(config, job):
+    return generate_beetmover_artifact_map(
+        config,
+        job,
+        **get_geckoview_template_vars(
+            config,
+            job["attributes"]["build_platform"],
+            job["maven-package"],
+            job["attributes"].get("update-channel"),
+        ),
+    )
+
+
+def get_geckoview_upstream_artifacts(config, job, package, platform=""):
     if not platform:
         platform = job["attributes"]["build_platform"]
     upstream_artifacts = generate_beetmover_upstream_artifacts(
@@ -35,7 +51,7 @@ def get_geckoview_upstream_artifacts(config, job, platform=""):
         job,
         platform="",
         **get_geckoview_template_vars(
-            config, platform, job["attributes"].get("update-channel")
+            config, platform, package, job["attributes"].get("update-channel")
         ),
     )
     return [
@@ -44,7 +60,7 @@ def get_geckoview_upstream_artifacts(config, job, platform=""):
     ]
 
 
-def get_geckoview_template_vars(config, platform, update_channel):
+def get_geckoview_template_vars(config, platform, package, update_channel):
     version_groups = re.match(r"(\d+).(\d+).*", config.params["version"])
     if version_groups:
         major_version, minor_version = version_groups.groups()
@@ -53,6 +69,7 @@ def get_geckoview_template_vars(config, platform, update_channel):
         "artifact_id": get_geckoview_artifact_id(
             config,
             platform,
+            package,
             update_channel,
         ),
         "build_date": config.params["moz_build_date"],
@@ -61,7 +78,7 @@ def get_geckoview_template_vars(config, platform, update_channel):
     }
 
 
-def get_geckoview_artifact_id(config, platform, update_channel=None):
+def get_geckoview_artifact_id(config, platform, package, update_channel=None):
     if update_channel == "release":
         update_channel = ""
     elif update_channel is not None:
@@ -71,4 +88,6 @@ def get_geckoview_artifact_id(config, platform, update_channel=None):
         # "nightly-{project}" for the update channel.  For other builds, the
         # update channel is not set, but the value is not substituted.
         update_channel = "-nightly-{}".format(config.params["project"])
-    return _ARTIFACT_ID_PER_PLATFORM[platform].format(update_channel=update_channel)
+    return _ARTIFACT_ID_PER_PLATFORM[platform].format(
+        update_channel=update_channel, package=package
+    )

taskcluster/gecko_taskgraph/util/signed_artifacts.py

@@ -186,7 +186,9 @@ def get_signed_artifacts(input, formats, behavior=None):
 
 
 def get_geckoview_artifacts_to_sign(config, job):
-    upstream_artifacts = get_geckoview_upstream_artifacts(config, job)
+    upstream_artifacts = []
+    for package in job["attributes"]["maven_packages"]:
+        upstream_artifacts += get_geckoview_upstream_artifacts(config, job, package)
     return [
         path
         for upstream_artifact in upstream_artifacts