Bug 1743687: Document what is needed to generate a new preloaded depot_tools zip r=firefox-build-system-reviewers,glandium

Depends on D129310

Differential Revision: https://phabricator.services.mozilla.com/D132590

taskcluster/ci/updatebot/kind.yml

@@ -83,6 +83,12 @@ jobs:
         worker:
             max-run-time: 3600
             taskcluster-proxy: true
+            # begin _GENERATE_DEPOT_TOOLS_BINARIES_
+            # artifacts:
+            #     - type: directory
+            #       name: public/depot_tools.zip
+            #       path: build\src\obj-build\depot_tools
+            # end _GENERATE_DEPOT_TOOLS_BINARIES_
             env:
                 TOOLTOOL_MANIFEST: "taskcluster/ci/updatebot/tooltool-manifests/updatebot.manifest"
                 MSYSTEM: "MINGW64"

taskcluster/ci/updatebot/tooltool-manifests/updatebot.manifest

@@ -18,11 +18,11 @@
     "unpack": true
   },
   {
-    "version": "Windows binaries for depot_tools retrieved with depot_tools revision 7b3351443198f4e7d068999b169385b55a43679a",
-    "filename": "depot_tools-preloaded-binaries-7b3351443198f4e7d068999b169385b55a43679a.zip",
-    "size": 315032144,
+    "version": "Windows binaries for depot_tools retrieved with depot_tools revision 7b3351443198f4e7d068999b169385b55a43679a on 2021-12-18",
+    "filename": "depot_tools-preloaded-binaries-7b3351443198f4e7d068999b169385b55a43679a-2021-12-18.zip",
+    "size": 315038144,
     "algorithm": "sha512",
-    "digest": "5f5071b05739dcb1b94fe94c1992d18b818bef05b8c47dd139941f0ac26e02a5b50b4efa5ba515a8227aead28ccd1b2dee563a553c42df369cfbb97051054b4d",
+    "digest": "250f3f3fcf631c53de1f505c53eb5ba782864fa92111eef0d710a1e7d0ad3da66db2fb8908d7c6d83e149674e976984d679f9a641eb9bea0b5c3d7b4f236920d",
     "visibility": "internal",
     "unpack": true
   }

taskcluster/docker/updatebot/windows-setup.sh

@@ -24,6 +24,23 @@ MOZ_FETCHES_PATH="$PWD"
 cd "$MOZ_FETCHES_DIR"
 mv depot_tools.git depot_tools
 
+
+# Generating a new version of the preloaded depot_tools download can be done by:
+# 1) Running the task, uncommenting the variable assignment below, uncommenting the 
+#    _GENERATE_DEPOT_TOOLS_BINARIES_ section in taskcluster/ci/updatebot/kind.yml,
+#    and ensuring that an angle update will actually take place (so it downloads the depot_tools)
+# 2) Downloading and sanity-checking the depot_tools-preloaded-binaries-GIT_HASH-DATE.zip artifact
+# 3) Adding it to tooltool
+# 4) Updating the updatebot manifest
+# Note that even for the same git revision the downloaded tools can change, so they are tagged
+# with both the git hash and the date it was generated
+
+# export GENERATE_DEPOT_TOOLS_BINARIES=1
+
+if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then
+    cp -r depot_tools depot_tools-from-git
+fi
+
 # Git is at /c/Program Files/Git/cmd/git.exe
 #  It's in PATH for this script (confusingly) but not in PATH so we need to add it
 export PATH="/c/Program Files/Git/cmd:$PATH"
@@ -80,6 +97,39 @@ export PYTHONUNBUFFERED=1
 cd "$UPDATEBOT_PATH"
 python3 -m poetry run python3 ./automation.py
 
+#########################################################
+if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then
+    # Artifacts
+
+    cd "$MOZ_FETCHES_PATH"
+    mv depot_tools depot_tools-from-tc
+
+    # Clean out unneeded files
+    # Need to use cmd because for some reason rm from bash throws 'Access Denied'
+    cmd '/c for /d /r %i in (*__pycache__) do rmdir /s /q %i'
+    rm -rf depot_tools-from-git/.git || true
+
+    # Delete the files that are already in git
+    find depot_tools-from-git -mindepth 1 -maxdepth 1 | sed s/depot_tools-from-git/depot_tools-from-tc/ | while read -r d; do rm -rf "$d"; done
+
+    # Make the artifact
+    rm -rf depot_tools-preloaded-binaries #remove it if it existed (i.e. we probably have one from tooltool already)
+    mv depot_tools-from-tc depot_tools-preloaded-binaries
+
+    # zip can't add symbolic links, and exits with an error code. || true avoids a script crash
+    zip -r depot_tools-preloaded-binaries.zip depot_tools-preloaded-binaries/ || true
+
+    # Convoluted way to get the git hash, because we don't have a .git directory
+    # Adding extra print statements just in case we need to debug it
+    GIT_HASH=$(grep depot_tools -A 1 "$GECKO_PATH/taskcluster/ci/fetch/updatebot.yml" | tee /dev/tty | grep revision | tee /dev/tty | awk -F': *' '{print $2}' | tee /dev/tty)
+    DATE=$(date -I)
+    mv depot_tools-preloaded-binaries.zip "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip"
+
+    # Put the artifact into the directory we will look for it
+    mkdir -p "$GECKO_PATH/obj-build/depot_tools" || true
+    mv "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip" "$GECKO_PATH/obj-build/depot_tools"
+fi
+
 #########################################################
 echo "Killing SQL Proxy"
 taskkill -f -im cloud_sql_proxy.exe || true