Bug 1387862 - Add initial support for ./mach lint -l yaml r=ahal,dustin

We should have CI Lint YAML files in the tree.

MozReview-Commit-ID: HYVWXzNnnzG

--HG--
rename : tools/lint/flake8_/__init__.py => tools/lint/yamllint_/__init__.py
extra : rebase_source : 54460179e51f6263099a6837e1111dcce05a9331

tools/lint/yaml.yml

@@ -0,0 +1,5 @@
+yamllint:
+    description: YAML linter
+    extensions: ['yml', 'yaml']
+    type: external
+    payload: yamllint_:lint

tools/lint/yamllint_/__init__.py

@@ -0,0 +1,154 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+import os
+import signal
+import subprocess
+
+import which
+from mozprocess import ProcessHandlerMixin
+
+from mozlint import result
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+YAMLLINT_REQUIREMENTS_PATH = os.path.join(here, 'yamllint_requirements.txt')
+
+
+YAMLLINT_INSTALL_ERROR = """
+Unable to install correct version of yamllint
+Try to install it manually with:
+    $ pip install -U --require-hashes -r {}
+""".strip().format(YAMLLINT_REQUIREMENTS_PATH)
+
+YAMLLINT_FORMAT_REGEX = re.compile(r'(.*):(.*):(.*): \[(error|warning)\] (.*) \((.*)\)$')
+
+results = []
+
+
+class YAMLLintProcess(ProcessHandlerMixin):
+    def __init__(self, config, *args, **kwargs):
+        self.config = config
+        kwargs['processOutputLine'] = [self.process_line]
+        ProcessHandlerMixin.__init__(self, *args, **kwargs)
+
+    def process_line(self, line):
+        try:
+            match = YAMLLINT_FORMAT_REGEX.match(line)
+            abspath, line, col, level, message, code = match.groups()
+        except AttributeError:
+            print('Unable to match yaml regex against output: {}'.format(line))
+            return
+
+        res = {'path': os.path.relpath(abspath, self.config['root']),
+               'message': message,
+               'level': level,
+               'lineno': line,
+               'column': col,
+               'rule': code,
+               }
+
+        results.append(result.from_config(self.config, **res))
+
+    def run(self, *args, **kwargs):
+        # protect against poor SIGINT handling. Handle it here instead
+        # so we can kill the process without a cryptic traceback.
+        orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+        ProcessHandlerMixin.run(self, *args, **kwargs)
+        signal.signal(signal.SIGINT, orig)
+
+
+def get_yamllint_binary():
+    """
+    Returns the path of the first yamllint binary available
+    if not found returns None
+    """
+    binary = os.environ.get('YAMLLINT')
+    if binary:
+        return binary
+
+    try:
+        return which.which('yamllint')
+    except which.WhichError:
+        return None
+
+
+def _run_pip(*args):
+    """
+    Helper function that runs pip with subprocess
+    """
+    try:
+        subprocess.check_output(['pip'] + list(args),
+                                stderr=subprocess.STDOUT)
+        return True
+    except subprocess.CalledProcessError as e:
+        print(e.output)
+        return False
+
+
+def reinstall_yamllint():
+    """
+    Try to install yamllint at the target version, returns True on success
+    otherwise prints the otuput of the pip command and returns False
+    """
+    if _run_pip('install', '-U',
+                '--require-hashes', '-r',
+                YAMLLINT_REQUIREMENTS_PATH):
+        return True
+
+    return False
+
+
+def run_process(config, cmd):
+    proc = YAMLLintProcess(config, cmd)
+    proc.run()
+    try:
+        proc.wait()
+    except KeyboardInterrupt:
+        proc.kill()
+
+
+def gen_yamllint_args(cmdargs, paths=None, conf_file=None):
+    args = cmdargs[:]
+    if isinstance(paths, basestring):
+        paths = [paths]
+    if conf_file:
+        return args + ['-c', conf_file] + paths
+    return args + paths
+
+
+def lint(files, config, **lintargs):
+
+    if not reinstall_yamllint():
+        print(YAMLLINT_INSTALL_ERROR)
+        return 1
+
+    binary = get_yamllint_binary()
+
+    cmdargs = [
+        binary,
+        '-f', 'parsable'
+    ]
+
+    config = config.copy()
+    config['root'] = lintargs['root']
+
+    # Run any paths with a .yamllint file in the directory separately so
+    # it gets picked up. This means only .yamllint files that live in
+    # directories that are explicitly included will be considered.
+    no_config = []
+    for f in files:
+        yamllint_config = os.path.join(f, '.yamllint')
+        if not os.path.isfile(yamllint_config):
+            no_config.append(f)
+            continue
+        run_process(config,
+                    gen_yamllint_args(cmdargs, conf_file=yamllint_config, paths=f))
+
+    if no_config:
+        run_process(config,
+                    gen_yamllint_args(cmdargs, paths=no_config))
+
+    return results

tools/lint/yamllint_/yamllint_requirements.txt

@@ -0,0 +1,21 @@
+yamllint==1.8.1 \
+    --hash=sha256:806b21828ca92fd6e9f20d8eccfa53035e38041854bb4d1b666f271a7788dbcf \
+    --hash=sha256:048743567ca9511e19222233ebb53795586a2cede07b79e801577e0a9b4f173c
+PyYAML==3.12 \
+    --hash=sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f \
+    --hash=sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736 \
+    --hash=sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269 \
+    --hash=sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8 \
+    --hash=sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4 \
+    --hash=sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1 \
+    --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \
+    --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \
+    --hash=sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3 \
+    --hash=sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8 \
+    --hash=sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6 \
+    --hash=sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca \
+    --hash=sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8 \
+    --hash=sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608 \
+    --hash=sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7
+pathspec==0.5.3 \
+    --hash=sha256:54478a66a360f4ebe4499c9235e4206fca5dec837b8e272d1ce37e0a626cc64d