diff --git a/build/sparse-profiles/docker-image b/build/sparse-profiles/docker-image index 9fb13295304d..aca31dd12236 100644 --- a/build/sparse-profiles/docker-image +++ b/build/sparse-profiles/docker-image @@ -12,8 +12,6 @@ path:python/mozbuild/mozbuild/action/tooltool.py path:testing/mozharness/external_tools/performance-artifact-schema.json path:testing/mozharness/external_tools/robustcheckout.py path:tools/lint/spell/codespell_requirements.txt -path:tools/lint/eslint/eslint-plugin-mozilla/manifest.tt -path:tools/lint/eslint/manifest.tt path:tools/lint/python/flake8_requirements.txt path:tools/lint/python/pylint_requirements.txt path:tools/lint/python/black_requirements.txt diff --git a/build/sparse-profiles/taskgraph b/build/sparse-profiles/taskgraph index 3c71497e9b8b..b188a7d92551 100644 --- a/build/sparse-profiles/taskgraph +++ b/build/sparse-profiles/taskgraph @@ -31,6 +31,10 @@ path:taskcluster/ # them all in. path:testing/config/tooltool-manifests/ path:testing/mozharness/ + +# for ESLint toolchains +path:package.json +path:package-lock.json path:tools/lint/ # for new-style try pushes diff --git a/docs/code-quality/lint/linters/eslint/enabling-rules.rst b/docs/code-quality/lint/linters/eslint/enabling-rules.rst index 49944a5ee0cd..ff6c40d17344 100644 --- a/docs/code-quality/lint/linters/eslint/enabling-rules.rst +++ b/docs/code-quality/lint/linters/eslint/enabling-rules.rst @@ -125,53 +125,3 @@ and when we upgrade the versions. The plugin can then be used with ESLint in the `normal way `_. - -Packaging node_modules ----------------------- - -For our continuous integration (CI) builders, we package ``node_modules`` for -both the top-level directory, and eslint-plugin-mozilla. These are uploaded to -our CI before the patch is released. - -Currently `Mark Banner (standard8) `_ -is the only person that does this regularly, and will be automatically added as -a blocking reviewer on patches that touch the relevant ``package.json`` files. - -A Release Engineering team member would likely have permissions to upload the -files as well. - -To upload the files, the process is: - -* Obtain ToolTool credentials for the public tooltool upload space. - - * Download the `taskcluster shell from here `_, - if you haven't already. - * Run the following command. This will open a page for you to log in, and - set environment variables for the following commands to use. - -.. code-block:: shell - - eval `TASKCLUSTER_ROOT_URL=https://firefox-ci-tc.services.mozilla.com taskcluster signin -s 'project:releng:services/tooltool/api/upload/public'` - -* Upload the eslint-plugin-mozilla packages: - -.. code-block:: shell - - cd tools/lint/eslint/eslint-plugin-mozilla/ - ./update.sh - - -* Upload the top-level packages: - -.. code-block:: shell - - cd .. - ./update.sh - - -* Add the changes to the commit that changes ``package.json``. - -The update scripts automatically clean out the ``node_modules`` directories, -removes the ``package-lock.json`` files, and then does a fresh installation. This -helps to ensure a "clean" directory with only the required modules, and an up to -date ``package-lock.json`` file. diff --git a/taskcluster/docker/lint/Dockerfile b/taskcluster/docker/lint/Dockerfile index e34d9730d7a9..7ff20319fb54 100644 --- a/taskcluster/docker/lint/Dockerfile +++ b/taskcluster/docker/lint/Dockerfile @@ -15,10 +15,6 @@ ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /build/tooltool.py ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /build/install-node.sh ADD system-setup.sh /tmp/system-setup.sh -# %include tools/lint/eslint/manifest.tt -ADD topsrcdir/tools/lint/eslint/manifest.tt /tmp/eslint.tt -# %include tools/lint/eslint/eslint-plugin-mozilla/manifest.tt -ADD topsrcdir/tools/lint/eslint/eslint-plugin-mozilla/manifest.tt /tmp/eslint-plugin-mozilla.tt # %include tools/lint/spell/codespell_requirements.txt ADD topsrcdir/tools/lint/spell/codespell_requirements.txt /tmp/codespell_requirements.txt # %include tools/lint/tox/tox_requirements.txt diff --git a/taskcluster/docker/lint/system-setup.sh b/taskcluster/docker/lint/system-setup.sh index 4ac14b8d80af..32dcc3f7f751 100644 --- a/taskcluster/docker/lint/system-setup.sh +++ b/taskcluster/docker/lint/system-setup.sh @@ -51,11 +51,6 @@ cd /build npm install -g yarn@1.22.18 -/build/tooltool.py fetch -m /tmp/eslint.tt -mv /build/node_modules /build/node_modules_eslint -/build/tooltool.py fetch -m /tmp/eslint-plugin-mozilla.tt -mv /build/node_modules /build/node_modules_eslint-plugin-mozilla - ### # fzf setup ### diff --git a/taskcluster/kinds/source-test/doc.yml b/taskcluster/kinds/source-test/doc.yml index cdac9a54a9e2..fca7aa07fee8 100644 --- a/taskcluster/kinds/source-test/doc.yml +++ b/taskcluster/kinds/source-test/doc.yml @@ -28,7 +28,7 @@ generate: using: run-task cwd: '{checkout}' command: > - cp -r /build/node_modules_eslint node_modules && + cp -r $MOZ_FETCHES_DIR/node_modules node_modules && ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && ./mach doc --outdir docs-out --no-open --no-serve --archive --check-num-warnings --dump-trees=/builds/worker/artifacts/trees.json @@ -38,6 +38,7 @@ generate: fetches: toolchain: - linux64-node + - node-modules upload: @@ -70,7 +71,7 @@ upload: using: run-task cwd: '{checkout}' command: > - cp -r /build/node_modules_eslint node_modules && + cp -r $MOZ_FETCHES_DIR/node_modules node_modules && ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && ./mach doc --upload --no-open --no-serve --write-url /builds/worker/firefox-source-docs-url.txt --dump-trees=/builds/worker/artifacts/trees.json @@ -82,3 +83,4 @@ upload: fetches: toolchain: - linux64-node + - node-modules diff --git a/taskcluster/kinds/source-test/mozlint.yml b/taskcluster/kinds/source-test/mozlint.yml index 2bde354de51f..4a26f7cc2d22 100644 --- a/taskcluster/kinds/source-test/mozlint.yml +++ b/taskcluster/kinds/source-test/mozlint.yml @@ -95,8 +95,7 @@ eslint: using: run-task cwd: '{checkout}' command: > - cp -r /build/node_modules_eslint node_modules && - cp -r /build/node_modules_eslint-plugin-mozilla/ tools/lint/eslint/eslint-plugin-mozilla/node_modules && + cp -r $MOZ_FETCHES_DIR/node_modules node_modules && ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && ./mach lint -v -l eslint -f treeherder -f json:/builds/worker/mozlint.json . @@ -122,6 +121,7 @@ eslint: fetches: toolchain: - linux64-node + - node-modules eslint-build: @@ -141,8 +141,7 @@ eslint-build: command: > export MOZ_XPT_ARTIFACTS_DIR="$MOZ_FETCHES_DIR/$MOZ_XPT_ARTIFACTS_DIRNAME" && find "$MOZ_XPT_ARTIFACTS_DIR" && - cp -r /build/node_modules_eslint node_modules && - cp -r /build/node_modules_eslint-plugin-mozilla/ tools/lint/eslint/eslint-plugin-mozilla/node_modules && + cp -r $MOZ_FETCHES_DIR/node_modules node_modules && ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && ./mach lint -v -l eslint -f treeherder --quiet -f json:/builds/worker/mozlint.json . --rule "mozilla/valid-services-property: error" --rule "mozilla/valid-ci-uses: error" @@ -157,6 +156,7 @@ eslint-build: dest: xpt_artifacts toolchain: - linux64-node + - node-modules stylelint: description: CSS lint check @@ -166,7 +166,9 @@ stylelint: using: run-task cwd: '{checkout}' command: > - cp -r /build/node_modules_eslint node_modules && + cp -r $MOZ_FETCHES_DIR/node_modules node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && ./mach lint -v -l stylelint -f treeherder -f json:/builds/worker/mozlint.json . when: files-changed: @@ -179,6 +181,7 @@ stylelint: fetches: toolchain: - linux64-node + - node-modules license: description: Check for license blocks in source files. diff --git a/taskcluster/kinds/source-test/node.yml b/taskcluster/kinds/source-test/node.yml index 7d9db46ee8c0..ecc6d0a3a3cc 100644 --- a/taskcluster/kinds/source-test/node.yml +++ b/taskcluster/kinds/source-test/node.yml @@ -167,7 +167,7 @@ eslint-plugin-mozilla: cache-dotcache: true cwd: '{checkout}' command: > - cp -r /build/node_modules_eslint-plugin-mozilla tools/lint/eslint/eslint-plugin-mozilla/node_modules && + cp -r $MOZ_FETCHES_DIR/eslint-plugin-mozilla/node_modules tools/lint/eslint/eslint-plugin-mozilla/node_modules && ./mach configure --disable-compile-environment && ./mach npm test --prefix tools/lint/eslint/eslint-plugin-mozilla when: @@ -176,6 +176,7 @@ eslint-plugin-mozilla: fetches: toolchain: - linux64-node + - eslint-plugin-mozilla design-tokens-tests: diff --git a/taskcluster/kinds/toolchain/misc.yml b/taskcluster/kinds/toolchain/misc.yml index f4f66ef1fe08..9ab739d273b1 100644 --- a/taskcluster/kinds/toolchain/misc.yml +++ b/taskcluster/kinds/toolchain/misc.yml @@ -356,6 +356,41 @@ browsertime: toolchain: - linux64-node +eslint-plugin-mozilla: + description: "npm install eslint-plugin-mozilla node_modules" + treeherder: + symbol: TL(epm) + run: + script: eslint-plugin-mozilla.sh + sparse-profile: null + resources: + - 'tools/lint/eslint/eslint-plugin-mozilla/package.json' + - 'tools/lint/eslint/eslint-plugin-mozilla/package-lock.json' + toolchain-artifact: public/build/eslint-plugin-mozilla.tar.zst + fetches: + toolchain: + - linux64-node + +node-modules: + description: "npm install top-level node_modules" + treeherder: + symbol: TL(node) + run: + script: node-modules.sh + sparse-profile: null + resources: + - 'package.json' + - 'package-lock.json' + - 'tools/lint/eslint/__init__.py' + - 'tools/lint/eslint/setup_helper.py' + - 'tools/lint/eslint/eslint-plugin-mozilla/package.json' + - 'tools/lint/eslint/eslint-plugin-mozilla/package-lock.json' + - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/package.json' + toolchain-artifact: public/build/node-modules.tar.zst + fetches: + toolchain: + - linux64-node + wrench-deps: description: "Downloads all the crates needed for building wrench" treeherder: diff --git a/taskcluster/scripts/misc/eslint-plugin-mozilla.sh b/taskcluster/scripts/misc/eslint-plugin-mozilla.sh new file mode 100755 index 000000000000..17841194d7ee --- /dev/null +++ b/taskcluster/scripts/misc/eslint-plugin-mozilla.sh @@ -0,0 +1,20 @@ +#!/bin/bash -vex + +set -x -e + +echo "running as" $(id) + +set -v + +cd $GECKO_PATH + +export PATH=$PATH:$MOZ_FETCHES_DIR/node/bin + +./mach configure --disable-compile-environment +./mach npm ci --prefix tools/lint/eslint/eslint-plugin-mozilla + +# We have tools/lint/eslint/eslint-plugin-mozilla/{node_modules,...} and want +# eslint-plugin-mozilla/{node_modules}. +mkdir -p /builds/worker/artifacts +cd tools/lint/eslint/ +tar caf /builds/worker/artifacts/eslint-plugin-mozilla.tar.zst eslint-plugin-mozilla diff --git a/taskcluster/scripts/misc/node-modules.sh b/taskcluster/scripts/misc/node-modules.sh new file mode 100755 index 000000000000..745f40327810 --- /dev/null +++ b/taskcluster/scripts/misc/node-modules.sh @@ -0,0 +1,20 @@ +#!/bin/bash -vex + +set -x -e + +echo "running as" $(id) + +set -v + +cd $GECKO_PATH + +export PATH=$PATH:$MOZ_FETCHES_DIR/node/bin + +./mach eslint --setup + +# Remove symlinks that we don't want to package. +rm node_modules/eslint-plugin-mozilla +rm node_modules/eslint-plugin-spidermonkey-js + +mkdir -p /builds/worker/artifacts +tar caf /builds/worker/artifacts/node-modules.tar.zst node_modules diff --git a/tools/lint/eslint/eslint-plugin-mozilla/update.sh b/tools/lint/eslint/eslint-plugin-mozilla/update.sh deleted file mode 100755 index 240cfdb9c242..000000000000 --- a/tools/lint/eslint/eslint-plugin-mozilla/update.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# Script to regenerate the npm packages used for eslint-plugin-mozilla by the builders. -# Requires - -# Force the scripts working directory to be projdir/tools/lint/eslint/eslint-plugin-mozilla. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $DIR - -if [ -z "$TASKCLUSTER_ACCESS_TOKEN" -o -z "$TASKCLUSTER_CLIENT_ID" -o -z "$TASKCLUSTER_ROOT_URL" ]; then - echo "Please ensure you have run the taskcluster shell correctly to set" - echo "the TASKCLUSTER_ACCESS_TOKEN, TASKCLUSTER_CLIENT_ID and" - echo "TASKCLUSTER_ROOT_URL environment variables." - echo "See https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint/enabling-rules.html" - exit 1; -fi - -echo "" -echo "Removing node_modules and package-lock.json..." -# Move to the top-level directory. -rm -rf node_modules -rm package-lock.json - -echo "Installing modules for eslint-plugin-mozilla..." -../../../../mach npm install - -echo "Creating eslint-plugin-mozilla.tar.gz..." -tar cvz -f eslint-plugin-mozilla.tar.gz node_modules - -echo "Adding eslint-plugin-mozilla.tar.gz to tooltool..." -rm -f manifest.tt -../../../../python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint-plugin-mozilla.tar.gz --url="https://tooltool.mozilla-releng.net/" - -echo "Uploading eslint-plugin-mozilla.tar.gz to tooltool..." -../../../../python/mozbuild/mozbuild/action/tooltool.py upload --message "node_modules folder update for tools/lint/eslint/eslint-plugin-mozilla" --url="https://tooltool.mozilla-releng.net/" - -echo "Cleaning up..." -rm eslint-plugin-mozilla.tar.gz - -echo "" -echo "Update complete, please commit and check in your changes." diff --git a/tools/lint/eslint/update.sh b/tools/lint/eslint/update.sh deleted file mode 100755 index 025f74713538..000000000000 --- a/tools/lint/eslint/update.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -# Script to regenerate the npm packages used for ESLint by the builders. -# Requires - -# Force the scripts working directory to be projdir/tools/lint/eslint. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $DIR - -if [ -z "$TASKCLUSTER_ACCESS_TOKEN" -o -z "$TASKCLUSTER_CLIENT_ID" -o -z "$TASKCLUSTER_ROOT_URL" ]; then - echo "Please ensure you have run the taskcluster shell correctly to set" - echo "the TASKCLUSTER_ACCESS_TOKEN, TASKCLUSTER_CLIENT_ID and" - echo "TASKCLUSTER_ROOT_URL environment variables." - echo "See https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint/enabling-rules.html" - exit 1; -fi - -echo "" -echo "Removing node_modules and package-lock.json..." -# Move to the top-level directory. -cd ../../../ -rm -rf node_modules/ -rm -rf tools/lint/eslint/eslint-plugin-mozilla/node_modules -rm package-lock.json - -echo "Installing eslint and external plugins..." -# ESLint and all _external_ plugins are listed in this directory's package.json, -# so a regular `npm install` will install them at the specified versions. -# The in-tree eslint-plugin-mozilla is kept out of this tooltool archive on -# purpose so that it can be changed by any developer without requiring tooltool -# access to make changes. -./mach npm install - -echo "Creating eslint.tar.gz..." -tar cvz --exclude=eslint-plugin-mozilla --exclude=eslint-plugin-spidermonkey-js -f eslint.tar.gz node_modules - -echo "Adding eslint.tar.gz to tooltool..." -rm tools/lint/eslint/manifest.tt -./python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint.tar.gz - -echo "Uploading eslint.tar.gz to tooltool..." -./python/mozbuild/mozbuild/action/tooltool.py upload --message "node_modules folder update for tools/lint/eslint" - -echo "Cleaning up..." -mv manifest.tt tools/lint/eslint/manifest.tt -rm eslint.tar.gz - -cd $DIR - -echo "" -echo "Update complete, please commit and check in your changes."