Bug 1623024 - mozlint: Add pylint as new linter r=linter-reviewers,ahal

Differential Revision: https://phabricator.services.mozilla.com/D79076

docs/code-quality/index.rst

@@ -100,7 +100,12 @@ In this document, we try to list these all tools.
      - `bug 1155970 <https://bugzilla.mozilla.org/show_bug.cgi?id=1155970>`_
      - :ref:`Flake8`
      - http://flake8.pycqa.org/
-   * - Python 2/3 compatibility check
+   * - pylint
+     -
+     - `bug 1623024 <https://bugzilla.mozilla.org/show_bug.cgi?id=1623024>`_
+     - :ref:`pylint`
+     - https://www.pylint.org/
+     * - Python 2/3 compatibility check
      -
      - `bug 1496527 <https://bugzilla.mozilla.org/show_bug.cgi?id=1496527>`_
      - :ref:`Python 2/3 compatibility check`

docs/code-quality/lint/linters/pylint.rst

@@ -0,0 +1,31 @@
+pylint
+======
+
+`pylint <https://www.pylint.org/>`__ is a popular linter for python developed by Logilab. It is now the default python
+linter in VS Code.
+
+Please note that we also have :ref:`Flake8` available as a linter.
+
+Run Locally
+-----------
+
+The mozlint integration of pylint can be run using mach:
+
+.. parsed-literal::
+
+    $ mach lint --linter pylint <file paths>
+
+
+
+Configuration
+-------------
+
+To enable pylint on new directory, add the path to the include
+section in the `pylint.yml <https://searchfox.org/mozilla-central/source/tools/lint/pylint.yml>`_ file.
+
+
+Sources
+-------
+
+* `Configuration (YAML) <https://searchfox.org/mozilla-central/source/tools/lint/pylint.yml>`_
+* `Source <https://searchfox.org/mozilla-central/source/tools/lint/python/pylint.py>`_

tools/lint/pylint.yml

@@ -0,0 +1,24 @@
+---
+pylint:
+    description: A second Python linter
+    include:
+        - configure.py
+        - client.py
+        - security/
+        - accessible/
+        - docs/
+        - dom/base/
+        - mozglue/
+    exclude:
+        - dom/bindings/Codegen.py
+        - dom/bindings/Configuration.py
+        - security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py
+        - security/manager/ssl/tests/unit/test_content_signing/pysign.py
+        - security/ct/tests/gtest/createSTHTestData.py
+    extensions: ['py']
+    support-files:
+        - '**/.pylint'
+        - 'tools/lint/python/pylint*'
+    type: external
+    payload: python.pylint:lint
+    setup: python.pylint:setup

tools/lint/python/pylint.py

@@ -0,0 +1,117 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+
+import signal
+import re
+
+from mozprocess import ProcessHandler
+
+from mozlint import result
+from mozlint.pathutils import expand_exclusions
+from mozlint.util import pip
+
+here = os.path.abspath(os.path.dirname(__file__))
+PYLINT_REQUIREMENTS_PATH = os.path.join(here, 'pylint_requirements.txt')
+
+PYLINT_NOT_FOUND = """
+Could not find pylint! Install pylint and try again.
+
+    $ pip install -U --require-hashes -r {}
+""".strip().format(PYLINT_REQUIREMENTS_PATH)
+
+
+PYLINT_INSTALL_ERROR = """
+Unable to install correct version of pylint
+Try to install it manually with:
+    $ pip install -U --require-hashes -r {}
+""".strip().format(PYLINT_REQUIREMENTS_PATH)
+
+
+class PylintProcess(ProcessHandler):
+    def __init__(self, config, *args, **kwargs):
+        self.config = config
+        kwargs["stream"] = False
+        kwargs["universal_newlines"] = True
+        ProcessHandler.__init__(self, *args, **kwargs)
+
+    def run(self, *args, **kwargs):
+        orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+        ProcessHandler.run(self, *args, **kwargs)
+        signal.signal(signal.SIGINT, orig)
+
+
+def setup(root, **lintargs):
+    if not pip.reinstall_program(PYLINT_REQUIREMENTS_PATH):
+        print(PYLINT_INSTALL_ERROR)
+        return 1
+
+
+def get_pylint_binary():
+    return "pylint"
+
+
+def run_process(config, cmd):
+    proc = PylintProcess(config, cmd)
+    proc.run()
+    try:
+        proc.wait()
+    except KeyboardInterrupt:
+        proc.kill()
+
+    return proc.output
+
+
+PYLINT_FORMAT_REGEX = re.compile(r'(.*):(.*): [(.*)] (.*)$')
+
+
+def parse_issues(log, config, issues_json, path):
+    results = []
+
+    try:
+        issues = json.loads(issues_json)
+    except json.decoder.JSONDecodeError:
+        log.debug("Could not parse the output:")
+        log.debug("pylint output: {}".format(issues_json))
+        return []
+
+    for issue in issues:
+        res = {
+            "path": issue["path"],
+            "level": issue["type"],
+            "lineno": issue["line"],
+            "column": issue["column"],
+            "message": issue["message"],
+            "rule": issue["message-id"],
+        }
+        results.append(result.from_config(config, **res))
+    return results
+
+
+def lint(paths, config, **lintargs):
+    log = lintargs['log']
+
+    binary = get_pylint_binary()
+
+    log = lintargs['log']
+    paths = list(expand_exclusions(paths, config, lintargs['root']))
+
+    cmd_args = [binary]
+    results = []
+
+    # list from https://code.visualstudio.com/docs/python/linting#_pylint
+    # And ignore a bit more elements
+    cmd_args += ["-fjson",
+                 "--disable=all",
+                 "--enable=F,E,unreachable,duplicate-key,unnecessary-semicolon,global-variable-not-assigned,unused-variable,binary-op-exception,bad-format-string,anomalous-backslash-in-string,bad-open-mode",  # NOQA: E501
+                 "--disable=import-error,no-member"]
+
+    base_command = cmd_args + paths
+    log.debug("Command: {}".format(' '.join(cmd_args)))
+    output = " ".join(run_process(config, base_command))
+    results = parse_issues(log, config, str(output), [])
+
+    return results

tools/lint/python/pylint_requirements.txt

@@ -0,0 +1,61 @@
+pylint==2.5.3 \
+    --hash=sha256:7dd78437f2d8d019717dbf287772d0b2dbdfd13fc016aa7faa08d67bccc46adc \
+    --hash=sha256:d0ece7d223fe422088b0e8f13fa0a1e8eb745ebffcb8ed53d3e95394b6101a1c
+toml==0.10.1 \
+    --hash=sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f \
+    --hash=sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88
+mccabe==0.6.1 \
+    --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
+    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+wrapt==1.12.1 \
+    --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7
+lazy-object-proxy==1.5.0 \
+    --hash=sha256:0aef3fa29f7d1194d6f8a99382b1b844e5a14d3bc1ef82c3b1c4fb7e7e2019bc \
+    --hash=sha256:159ae2bbb4dc3ba506aeba868d14e56a754c0be402d1f0d7fdb264e0bdf2b095 \
+    --hash=sha256:161a68a427022bf13e249458be2cb8da56b055988c584d372a917c665825ae9a \
+    --hash=sha256:2d58f0e6395bf41087a383a48b06b42165f3b699f1aa41ba201db84ab77be63d \
+    --hash=sha256:311c9d1840042fc8e2dd80fc80272a7ea73e7646745556153c9cda85a4628b18 \
+    --hash=sha256:35c3ad7b7f7d5d4a54a80f0ff5a41ab186237d6486843f8dde00c42cfab33905 \
+    --hash=sha256:459ef557e669d0046fe2b92eb4822c097c00b5ef9d11df0f9bd7d4267acdfc52 \
+    --hash=sha256:4a50513b6be001b9b7be2c435478fe9669249c77c241813907a44cda1fcd03f4 \
+    --hash=sha256:51035b175740c44707694c521560b55b66da9d5a7c545cf22582bc02deb61664 \
+    --hash=sha256:96f2cdb35bdfda10e075f12892a42cff5179bbda698992b845f36c5e92755d33 \
+    --hash=sha256:a0aed261060cd0372abf08d16399b1224dbb5b400312e6b00f2b23eabe1d4e96 \
+    --hash=sha256:a6052c4c7d95de2345d9c58fc0fe34fff6c27a8ed8550dafeb18ada84406cc99 \
+    --hash=sha256:cbf1354292a4f7abb6a0188f74f5e902e4510ebad105be1dbc4809d1ed92f77e \
+    --hash=sha256:da82b2372f5ded8806eaac95b19af89a7174efdb418d4e7beb0c6ab09cee7d95 \
+    --hash=sha256:dd89f466c930d7cfe84c94b5cbe862867c88b269f23e5aa61d40945e0d746f54 \
+    --hash=sha256:e3183fbeb452ec11670c2d9bfd08a57bc87e46856b24d1c335f995239bedd0e1 \
+    --hash=sha256:e9a571e7168076a0d5ecaabd91e9032e86d815cca3a4bf0dafead539ef071aa5 \
+    --hash=sha256:ec6aba217d0c4f71cbe48aea962a382dedcd111f47b55e8b58d4aaca519bd360
+astroid==2.4.2 \
+    --hash=sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703 \
+    --hash=sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386
+isort==4.3.21 \
+    --hash=sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1 \
+    --hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd
+typed-ast==1.4.1 \
+    --hash=sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355 \
+    --hash=sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919 \
+    --hash=sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa \
+    --hash=sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652 \
+    --hash=sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75 \
+    --hash=sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01 \
+    --hash=sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d \
+    --hash=sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1 \
+    --hash=sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907 \
+    --hash=sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c \
+    --hash=sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3 \
+    --hash=sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b \
+    --hash=sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614 \
+    --hash=sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb \
+    --hash=sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b \
+    --hash=sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41 \
+    --hash=sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6 \
+    --hash=sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34 \
+    --hash=sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe \
+    --hash=sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4 \
+    --hash=sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7

tools/lint/test/files/pylint/bad.py

@@ -0,0 +1,5 @@
+def foo():
+    useless_var = 1
+    useless_var = true
+    return "true"
+    print("unreachable")

tools/lint/test/files/pylint/good.py

@@ -0,0 +1,3 @@
+def foo():
+    a = 1 + 1
+    return a

tools/lint/test/python.ini

@@ -24,3 +24,7 @@ skip-if = os == "win" || os == "mac"  # only installed on Linux
 skip-if = os == "win" || os == "mac"  # only installed on Linux
 [test_clang_format.py]
 skip-if = os == "win" || os == "mac"  # only installed on Linux
+[test_pylint.py]
+skip-if = os == "win" || os == "mac"  # only installed on linux
+requirements = tools/lint/python/pylint_requirements.txt
+

tools/lint/test/test_pylint.py

@@ -0,0 +1,24 @@
+import mozunit
+
+LINTER = 'pylint'
+
+
+def test_lint_single_file(lint, paths):
+    results = lint(paths('bad.py'))
+    assert len(results) == 3
+    assert results[0].rule == 'E0602'
+    assert results[1].rule == 'W0101'
+    assert results[1].lineno == 5
+
+    # run lint again to make sure the previous results aren't counted twice
+    results = lint(paths('bad.py'))
+    assert len(results) == 3
+
+
+def test_lint_single_file_good(lint, paths):
+    results = lint(paths('good.py'))
+    assert len(results) == 0
+
+
+if __name__ == '__main__':
+    mozunit.main()