Bug 1920970 - Move ESLint node_modules bundles away from tooltool and into the toolchain. r=jcristau

Differential Revision: https://phabricator.services.mozilla.com/D223647

build/sparse-profiles/docker-image

@@ -12,8 +12,6 @@ path:python/mozbuild/mozbuild/action/tooltool.py
 path:testing/mozharness/external_tools/performance-artifact-schema.json
 path:testing/mozharness/external_tools/robustcheckout.py
 path:tools/lint/spell/codespell_requirements.txt
-path:tools/lint/eslint/eslint-plugin-mozilla/manifest.tt
-path:tools/lint/eslint/manifest.tt
 path:tools/lint/python/flake8_requirements.txt
 path:tools/lint/python/pylint_requirements.txt
 path:tools/lint/python/black_requirements.txt

build/sparse-profiles/taskgraph

@@ -31,6 +31,10 @@ path:taskcluster/
 # them all in.
 path:testing/config/tooltool-manifests/
 path:testing/mozharness/
+
+# for ESLint toolchains
+path:package.json
+path:package-lock.json
 path:tools/lint/
 
 # for new-style try pushes

docs/code-quality/lint/linters/eslint/enabling-rules.rst

@@ -125,53 +125,3 @@ and when we upgrade the versions.
 
 The plugin can then be used with ESLint in the
 `normal way <https://eslint.org/docs/latest/use/configure/plugins>`_.
-
-Packaging node_modules
-----------------------
-
-For our continuous integration (CI) builders, we package ``node_modules`` for
-both the top-level directory, and eslint-plugin-mozilla. These are uploaded to
-our CI before the patch is released.
-
-Currently `Mark Banner (standard8) <https://people.mozilla.org/s?query=standard8>`_
-is the only person that does this regularly, and will be automatically added as
-a blocking reviewer on patches that touch the relevant ``package.json`` files.
-
-A Release Engineering team member would likely have permissions to upload the
-files as well.
-
-To upload the files, the process is:
-
-* Obtain ToolTool credentials for the public tooltool upload space.
-
-    * Download the `taskcluster shell from here <https://github.com/taskcluster/taskcluster/tree/main/clients/client-shell>`_,
-      if you haven't already.
-    * Run the following command. This will open a page for you to log in, and
-      set environment variables for the following commands to use.
-
-.. code-block:: shell
-
-    eval `TASKCLUSTER_ROOT_URL=https://firefox-ci-tc.services.mozilla.com taskcluster signin -s 'project:releng:services/tooltool/api/upload/public'`
-
-* Upload the eslint-plugin-mozilla packages:
-
-.. code-block:: shell
-
-    cd tools/lint/eslint/eslint-plugin-mozilla/
-    ./update.sh
-    <follow the instructions>
-
-* Upload the top-level packages:
-
-.. code-block:: shell
-
-    cd ..
-    ./update.sh
-    <follow the instructions>
-
-* Add the changes to the commit that changes ``package.json``.
-
-The update scripts automatically clean out the ``node_modules`` directories,
-removes the ``package-lock.json`` files, and then does a fresh installation. This
-helps to ensure a "clean" directory with only the required modules, and an up to
-date ``package-lock.json`` file.

taskcluster/docker/lint/Dockerfile

@@ -15,10 +15,6 @@ ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /build/tooltool.py
 ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /build/install-node.sh
 
 ADD system-setup.sh /tmp/system-setup.sh
-# %include tools/lint/eslint/manifest.tt
-ADD topsrcdir/tools/lint/eslint/manifest.tt /tmp/eslint.tt
-# %include tools/lint/eslint/eslint-plugin-mozilla/manifest.tt
-ADD topsrcdir/tools/lint/eslint/eslint-plugin-mozilla/manifest.tt /tmp/eslint-plugin-mozilla.tt
 # %include tools/lint/spell/codespell_requirements.txt
 ADD topsrcdir/tools/lint/spell/codespell_requirements.txt /tmp/codespell_requirements.txt
 # %include tools/lint/tox/tox_requirements.txt

taskcluster/docker/lint/system-setup.sh

@@ -51,11 +51,6 @@ cd /build
 
 npm install -g yarn@1.22.18
 
-/build/tooltool.py fetch -m /tmp/eslint.tt
-mv /build/node_modules /build/node_modules_eslint
-/build/tooltool.py fetch -m /tmp/eslint-plugin-mozilla.tt
-mv /build/node_modules /build/node_modules_eslint-plugin-mozilla
-
 ###
 # fzf setup
 ###

taskcluster/kinds/source-test/doc.yml

@@ -28,7 +28,7 @@ generate:
         using: run-task
         cwd: '{checkout}'
         command: >
-            cp -r /build/node_modules_eslint node_modules &&
+            cp -r $MOZ_FETCHES_DIR/node_modules node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules &&
             ./mach doc --outdir docs-out --no-open --no-serve --archive --check-num-warnings --dump-trees=/builds/worker/artifacts/trees.json
@@ -38,6 +38,7 @@ generate:
     fetches:
         toolchain:
             - linux64-node
+            - node-modules
 
 
 upload:
@@ -70,7 +71,7 @@ upload:
         using: run-task
         cwd: '{checkout}'
         command: >
-            cp -r /build/node_modules_eslint node_modules &&
+            cp -r $MOZ_FETCHES_DIR/node_modules node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules &&
             ./mach doc --upload --no-open --no-serve --write-url /builds/worker/firefox-source-docs-url.txt --dump-trees=/builds/worker/artifacts/trees.json
@@ -82,3 +83,4 @@ upload:
     fetches:
         toolchain:
             - linux64-node
+            - node-modules

taskcluster/kinds/source-test/mozlint.yml

@@ -95,8 +95,7 @@ eslint:
         using: run-task
         cwd: '{checkout}'
         command: >
-            cp -r /build/node_modules_eslint node_modules &&
-            cp -r /build/node_modules_eslint-plugin-mozilla/ tools/lint/eslint/eslint-plugin-mozilla/node_modules &&
+            cp -r $MOZ_FETCHES_DIR/node_modules node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules &&
             ./mach lint -v -l eslint -f treeherder -f json:/builds/worker/mozlint.json .
@@ -122,6 +121,7 @@ eslint:
     fetches:
         toolchain:
             - linux64-node
+            - node-modules
 
 
 eslint-build:
@@ -141,8 +141,7 @@ eslint-build:
         command: >
             export MOZ_XPT_ARTIFACTS_DIR="$MOZ_FETCHES_DIR/$MOZ_XPT_ARTIFACTS_DIRNAME" &&
             find "$MOZ_XPT_ARTIFACTS_DIR" &&
-            cp -r /build/node_modules_eslint node_modules &&
-            cp -r /build/node_modules_eslint-plugin-mozilla/ tools/lint/eslint/eslint-plugin-mozilla/node_modules &&
+            cp -r $MOZ_FETCHES_DIR/node_modules node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules &&
             ./mach lint -v -l eslint -f treeherder --quiet -f json:/builds/worker/mozlint.json . --rule "mozilla/valid-services-property: error" --rule "mozilla/valid-ci-uses: error"
@@ -157,6 +156,7 @@ eslint-build:
               dest: xpt_artifacts
         toolchain:
             - linux64-node
+            - node-modules
 
 stylelint:
     description: CSS lint check
@@ -166,7 +166,9 @@ stylelint:
         using: run-task
         cwd: '{checkout}'
         command: >
-            cp -r /build/node_modules_eslint node_modules &&
+            cp -r $MOZ_FETCHES_DIR/node_modules node_modules &&
+            ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules &&
+            ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules &&
             ./mach lint -v -l stylelint -f treeherder -f json:/builds/worker/mozlint.json .
     when:
         files-changed:
@@ -179,6 +181,7 @@ stylelint:
     fetches:
         toolchain:
             - linux64-node
+            - node-modules
 
 license:
     description: Check for license blocks in source files.

taskcluster/kinds/source-test/node.yml

@@ -167,7 +167,7 @@ eslint-plugin-mozilla:
         cache-dotcache: true
         cwd: '{checkout}'
         command: >
-            cp -r /build/node_modules_eslint-plugin-mozilla tools/lint/eslint/eslint-plugin-mozilla/node_modules &&
+            cp -r $MOZ_FETCHES_DIR/eslint-plugin-mozilla/node_modules tools/lint/eslint/eslint-plugin-mozilla/node_modules &&
             ./mach configure --disable-compile-environment &&
             ./mach npm test --prefix tools/lint/eslint/eslint-plugin-mozilla
     when:
@@ -176,6 +176,7 @@ eslint-plugin-mozilla:
     fetches:
         toolchain:
             - linux64-node
+            - eslint-plugin-mozilla
 
 
 design-tokens-tests:

taskcluster/kinds/toolchain/misc.yml

@@ -356,6 +356,41 @@ browsertime:
         toolchain:
             - linux64-node
 
+eslint-plugin-mozilla:
+    description: "npm install eslint-plugin-mozilla node_modules"
+    treeherder:
+        symbol: TL(epm)
+    run:
+        script: eslint-plugin-mozilla.sh
+        sparse-profile: null
+        resources:
+            - 'tools/lint/eslint/eslint-plugin-mozilla/package.json'
+            - 'tools/lint/eslint/eslint-plugin-mozilla/package-lock.json'
+        toolchain-artifact: public/build/eslint-plugin-mozilla.tar.zst
+    fetches:
+        toolchain:
+            - linux64-node
+
+node-modules:
+    description: "npm install top-level node_modules"
+    treeherder:
+        symbol: TL(node)
+    run:
+        script: node-modules.sh
+        sparse-profile: null
+        resources:
+            - 'package.json'
+            - 'package-lock.json'
+            - 'tools/lint/eslint/__init__.py'
+            - 'tools/lint/eslint/setup_helper.py'
+            - 'tools/lint/eslint/eslint-plugin-mozilla/package.json'
+            - 'tools/lint/eslint/eslint-plugin-mozilla/package-lock.json'
+            - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/package.json'
+        toolchain-artifact: public/build/node-modules.tar.zst
+    fetches:
+        toolchain:
+            - linux64-node
+
 wrench-deps:
     description: "Downloads all the crates needed for building wrench"
     treeherder:

taskcluster/scripts/misc/eslint-plugin-mozilla.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+set -v
+
+cd $GECKO_PATH
+
+export PATH=$PATH:$MOZ_FETCHES_DIR/node/bin
+
+./mach configure --disable-compile-environment
+./mach npm ci --prefix tools/lint/eslint/eslint-plugin-mozilla
+
+# We have tools/lint/eslint/eslint-plugin-mozilla/{node_modules,...} and want
+# eslint-plugin-mozilla/{node_modules}.
+mkdir -p /builds/worker/artifacts
+cd tools/lint/eslint/
+tar caf /builds/worker/artifacts/eslint-plugin-mozilla.tar.zst eslint-plugin-mozilla

taskcluster/scripts/misc/node-modules.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+set -v
+
+cd $GECKO_PATH
+
+export PATH=$PATH:$MOZ_FETCHES_DIR/node/bin
+
+./mach eslint --setup
+
+# Remove symlinks that we don't want to package.
+rm node_modules/eslint-plugin-mozilla
+rm node_modules/eslint-plugin-spidermonkey-js
+
+mkdir -p /builds/worker/artifacts
+tar caf /builds/worker/artifacts/node-modules.tar.zst node_modules

tools/lint/eslint/eslint-plugin-mozilla/update.sh

@@ -1,40 +0,0 @@
-#!/bin/sh
-# Script to regenerate the npm packages used for eslint-plugin-mozilla by the builders.
-# Requires
-
-# Force the scripts working directory to be projdir/tools/lint/eslint/eslint-plugin-mozilla.
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd $DIR
-
-if [ -z "$TASKCLUSTER_ACCESS_TOKEN" -o -z "$TASKCLUSTER_CLIENT_ID" -o -z "$TASKCLUSTER_ROOT_URL" ]; then
-  echo "Please ensure you have run the taskcluster shell correctly to set"
-  echo "the TASKCLUSTER_ACCESS_TOKEN, TASKCLUSTER_CLIENT_ID and"
-  echo "TASKCLUSTER_ROOT_URL environment variables."
-  echo "See https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint/enabling-rules.html"
-  exit 1;
-fi
-
-echo ""
-echo "Removing node_modules and package-lock.json..."
-# Move to the top-level directory.
-rm -rf node_modules
-rm package-lock.json
-
-echo "Installing modules for eslint-plugin-mozilla..."
-../../../../mach npm install
-
-echo "Creating eslint-plugin-mozilla.tar.gz..."
-tar cvz -f eslint-plugin-mozilla.tar.gz node_modules
-
-echo "Adding eslint-plugin-mozilla.tar.gz to tooltool..."
-rm -f manifest.tt
-../../../../python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint-plugin-mozilla.tar.gz --url="https://tooltool.mozilla-releng.net/"
-
-echo "Uploading eslint-plugin-mozilla.tar.gz to tooltool..."
-../../../../python/mozbuild/mozbuild/action/tooltool.py upload --message "node_modules folder update for tools/lint/eslint/eslint-plugin-mozilla" --url="https://tooltool.mozilla-releng.net/"
-
-echo "Cleaning up..."
-rm eslint-plugin-mozilla.tar.gz
-
-echo ""
-echo "Update complete, please commit and check in your changes."

tools/lint/eslint/update.sh

@@ -1,50 +0,0 @@
-#!/bin/sh
-# Script to regenerate the npm packages used for ESLint by the builders.
-# Requires
-
-# Force the scripts working directory to be projdir/tools/lint/eslint.
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd $DIR
-
-if [ -z "$TASKCLUSTER_ACCESS_TOKEN" -o -z "$TASKCLUSTER_CLIENT_ID" -o -z "$TASKCLUSTER_ROOT_URL" ]; then
-  echo "Please ensure you have run the taskcluster shell correctly to set"
-  echo "the TASKCLUSTER_ACCESS_TOKEN, TASKCLUSTER_CLIENT_ID and"
-  echo "TASKCLUSTER_ROOT_URL environment variables."
-  echo "See https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint/enabling-rules.html"
-  exit 1;
-fi
-
-echo ""
-echo "Removing node_modules and package-lock.json..."
-# Move to the top-level directory.
-cd ../../../
-rm -rf node_modules/
-rm -rf tools/lint/eslint/eslint-plugin-mozilla/node_modules
-rm package-lock.json
-
-echo "Installing eslint and external plugins..."
-# ESLint and all _external_ plugins are listed in this directory's package.json,
-# so a regular `npm install` will install them at the specified versions.
-# The in-tree eslint-plugin-mozilla is kept out of this tooltool archive on
-# purpose so that it can be changed by any developer without requiring tooltool
-# access to make changes.
-./mach npm install
-
-echo "Creating eslint.tar.gz..."
-tar cvz --exclude=eslint-plugin-mozilla --exclude=eslint-plugin-spidermonkey-js -f eslint.tar.gz node_modules
-
-echo "Adding eslint.tar.gz to tooltool..."
-rm tools/lint/eslint/manifest.tt
-./python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint.tar.gz
-
-echo "Uploading eslint.tar.gz to tooltool..."
-./python/mozbuild/mozbuild/action/tooltool.py upload --message "node_modules folder update for tools/lint/eslint"
-
-echo "Cleaning up..."
-mv manifest.tt tools/lint/eslint/manifest.tt
-rm eslint.tar.gz
-
-cd $DIR
-
-echo ""
-echo "Update complete, please commit and check in your changes."