servo: Merge #6279 - Add flake8 to the tidy process for Python files (from frewsxcv:flake8); r=jdm

Fixes #6236

Also included in this commit are the changes need to make flake8 pass
for the existing python file

Source-Repo: https://github.com/servo/servo
Source-Revision: ccfe29d8f284dedc0101045d574a98fb1f69aa62

servo/etc/servo_gdb.py

@@ -7,19 +7,22 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-# A set of simple pretty printers for gdb to make debugging Servo a bit easier.
+"""
+A set of simple pretty printers for gdb to make debugging Servo a bit easier.
 
-# To load these, you need to add something like the following
-# to your .gdbinit file.
-#python
-#import sys
-#sys.path.insert(0, '/home/<path to git checkout>/servo/src/etc')
-#import servo_gdb
-#servo_gdb.register_printers(None)
-#end
+To load these, you need to add something like the following to your .gdbinit file:
+
+python
+import sys
+sys.path.insert(0, '/home/<path to git checkout>/servo/src/etc')
+import servo_gdb
+servo_gdb.register_printers(None)
+end
+"""
 
 import gdb
 
+
 # Print Au in both raw value and CSS pixels
 class AuPrinter:
     def __init__(self, val):
@@ -27,9 +30,10 @@ class AuPrinter:
 
     def to_string(self):
         i32_type = gdb.lookup_type("i32")
-        au = self.val.cast(i32_type);
+        au = self.val.cast(i32_type)
         return "{0}px".format(au / 60.0)
 
+
 # Print a U8 bitfield as binary
 class BitFieldU8Printer:
     def __init__(self, val):
@@ -37,9 +41,10 @@ class BitFieldU8Printer:
 
     def to_string(self):
         u8_type = gdb.lookup_type("u8")
-        value = self.val.cast(u8_type);
+        value = self.val.cast(u8_type)
         return "[{0:#010b}]".format(int(value))
 
+
 # Print a struct with fields as children
 class ChildPrinter:
     def __init__(self, val):
@@ -48,12 +53,13 @@ class ChildPrinter:
     def children(self):
         children = []
         for f in self.val.type.fields():
-            children.append( (f.name, self.val[f.name]) )
+            children.append((f.name, self.val[f.name]))
         return children
 
     def to_string(self):
         return None
 
+
 # Allow a trusted node to be dereferenced in the debugger
 class TrustedNodeAddressPrinter:
     def __init__(self, val):
@@ -67,6 +73,7 @@ class TrustedNodeAddressPrinter:
     def to_string(self):
         return self.val.address
 
+
 # Extract a node type ID from enum
 class NodeTypeIdPrinter:
     def __init__(self, val):
@@ -75,9 +82,10 @@ class NodeTypeIdPrinter:
     def to_string(self):
         u8_ptr_type = gdb.lookup_type("u8").pointer()
         enum_0 = self.val.address.cast(u8_ptr_type).dereference()
-        enum_type = self.val.type.fields()[int(enum_0)].type;
+        enum_type = self.val.type.fields()[int(enum_0)].type
         return str(enum_type).lstrip('struct ')
 
+
 # Printer for std::Option<>
 class OptionPrinter:
     def __init__(self, val):
@@ -111,6 +119,7 @@ class OptionPrinter:
     def to_string(self):
         return None
 
+
 # Useful for debugging when type is unknown
 class TestPrinter:
     def __init__(self, val):
@@ -129,13 +138,15 @@ type_map = [
     ('Option', OptionPrinter),
 ]
 
-def lookup_servo_type (val):
+
+def lookup_servo_type(val):
     val_type = str(val.type)
     for (type_name, printer) in type_map:
-        if val_type == type_name or val_type.endswith("::"+type_name):
+        if val_type == type_name or val_type.endswith("::" + type_name):
             return printer(val)
     return None
-    #return TestPrinter(val)
+    # return TestPrinter(val)
+
 
 def register_printers(obj):
     gdb.pretty_printers.append(lookup_servo_type)

servo/python/licenseck.py

@@ -59,4 +59,4 @@ licenses = [
 # except according to those terms.
 """,
 
-]
+]  # noqa: Indicate to flake8 that we do not want to check indentation here

servo/python/mach_bootstrap.py

@@ -98,7 +98,7 @@ def bootstrap(topdir):
 
     for category, meta in CATEGORIES.items():
         mach.define_category(category, meta['short'], meta['long'],
-            meta['priority'])
+                             meta['priority'])
 
     for path in MACH_MODULES:
         mach.load_commands_from_file(os.path.join(topdir, path))

servo/python/servo/bootstrap_commands.py

@@ -40,7 +40,8 @@ def download(desc, src, dst):
         with open(dst, 'wb') as fd:
             while True:
                 chunk = resp.read(chunk_size)
-                if not chunk: break
+                if not chunk:
+                    break
                 recved += len(chunk)
                 if not dumb:
                     pct = recved * 100.0 / fsize
@@ -61,6 +62,7 @@ def download(desc, src, dst):
 
         sys.exit(1)
 
+
 def extract(src, dst, movedir=None):
     tarfile.open(src).extractall(dst)
 

servo/python/servo/build_commands.py

@@ -58,13 +58,14 @@ def notify_win(title, text):
     FLASHW_TIMERNOFG = 0x0C
 
     params = FLASHWINDOW(sizeof(FLASHWINDOW),
-                        windll.kernel32.GetConsoleWindow(),
-                        FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0)
+                         windll.kernel32.GetConsoleWindow(),
+                         FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0)
     FlashWindowEx(params)
 
 
 def notify_darwin(title, text):
-    import Foundation, objc
+    import Foundation
+    import objc
 
     NSUserNotification = objc.lookUpClass("NSUserNotification")
     NSUserNotificationCenter = objc.lookUpClass("NSUserNotificationCenter")
@@ -291,7 +292,7 @@ class MachCommands(CommandBase):
             opts += ["--release"]
 
         opts += ["--target", "arm-linux-androideabi"]
-        env=self.build_env(gonk=True)
+        env = self.build_env(gonk=True)
         build_start = time()
         with cd(path.join("ports", "gonk")):
             ret = subprocess.call(["cargo", "build"] + opts, env=env)
@@ -304,7 +305,6 @@ class MachCommands(CommandBase):
 
         return ret
 
-
     @Command('build-tests',
              description='Build the Servo test suites',
              category='build')
@@ -329,7 +329,6 @@ class MachCommands(CommandBase):
     @CommandArgument('--verbose', '-v',
                      action='store_true',
                      help='Print verbose output')
-
     @CommandArgument('params', nargs='...',
                      help="Command-line arguments to be passed through to Cargo")
     def clean(self, manifest_path, params, verbose=False):

servo/python/servo/command_base.py

@@ -165,7 +165,7 @@ class CommandBase(object):
 
         print("The %s profile is not built. Please run './mach build%s' "
               "and try again." % ("release" if release else "dev",
-                                   " --release" if release else ""))
+                                  " --release" if release else ""))
         sys.exit()
 
     def build_env(self, gonk=False, hosts_file_path=None):

servo/python/servo/devenv_commands.py

@@ -36,8 +36,8 @@ class MachCommands(CommandBase):
 
         if self.context.topdir == getcwd():
             with cd(path.join('components', 'servo')):
-                return subprocess.call(["cargo"] + params,
-                               env=self.build_env())
+                return subprocess.call(
+                    ["cargo"] + params, env=self.build_env())
         return subprocess.call(['cargo'] + params,
                                env=self.build_env())
 
@@ -51,7 +51,7 @@ class MachCommands(CommandBase):
         '--package', '-p', default=None,
         help='Updates selected package')
     @CommandArgument(
-        '--all-packages','-a',action='store_true',
+        '--all-packages', '-a', action='store_true',
         help='Updates all packages')
     def cargo_update(self, params=None, package=None, all_packages=None):
         self.update_cargo(params, package, all_packages)
@@ -63,10 +63,10 @@ class MachCommands(CommandBase):
         'params', default=None, nargs='...',
         help='Command-line arguments to be passed through to cargo update')
     @CommandArgument(
-        '--package','-p',default=None,
+        '--package', '-p', default=None,
         help='Updates selected package')
     @CommandArgument(
-        '--all-packages','-a',action='store_true',
+        '--all-packages', '-a', action='store_true',
         help='Updates all packages')
     def update_cargo(self, params=None, package=None, all_packages=None):
         if not params:

servo/python/servo/post_build_commands.py

@@ -12,7 +12,6 @@ from __future__ import print_function, unicode_literals
 import os
 import os.path as path
 import subprocess
-import sys
 from shutil import copytree, rmtree, copy2
 
 from mach.registrar import Registrar
@@ -49,7 +48,7 @@ class MachCommands(CommandBase):
                           'debugger being used. The following arguments '
                           'have no effect without this.')
     @CommandArgument('--debugger', default=None, type=str,
-        help='Name of debugger to use.')
+                     help='Name of debugger to use.')
     @CommandArgument(
         'params', nargs='...',
         help="Command-line arguments to be passed through to Servo")
@@ -74,8 +73,8 @@ class MachCommands(CommandBase):
                 return 1
 
             # Prepend the debugger args.
-            args = ([self.debuggerInfo.path] + self.debuggerInfo.args
-                    + args + params)
+            args = ([self.debuggerInfo.path] + self.debuggerInfo.args +
+                    args + params)
         else:
             args = args + params
 
@@ -84,7 +83,7 @@ class MachCommands(CommandBase):
         except OSError as e:
             if e.errno == 2:
                 print("Servo Binary can't be found! Run './mach build'"
-                     " and try again!")
+                      " and try again!")
             else:
                 raise e
 
@@ -132,8 +131,7 @@ class MachCommands(CommandBase):
         help="Command-line arguments to be passed through to cargo doc")
     def doc(self, params):
         self.ensure_bootstrapped()
-        if not path.exists(path.join(
-               self.config["tools"]["rust-root"], "doc")):
+        if not path.exists(path.join(self.config["tools"]["rust-root"], "doc")):
             Registrar.dispatch("bootstrap-rust-docs", context=self.context)
         rust_docs = path.join(self.config["tools"]["rust-root"], "doc")
         docs = path.join(

servo/python/servo/testing_commands.py

@@ -281,7 +281,6 @@ class MachCommands(CommandBase):
         execfile(run_file, run_globals)
         return run_globals["update_tests"](**kwargs)
 
-
     def ensure_wpt_virtualenv(self):
         virtualenv_path = path.join("tests", "wpt", "_virtualenv")
         python = self.get_exec("python2", "python")
@@ -295,8 +294,8 @@ class MachCommands(CommandBase):
         execfile(activate_path, dict(__file__=activate_path))
 
         try:
-            import wptrunner
-            from wptrunner.browsers import servo
+            import wptrunner  # noqa
+            from wptrunner.browsers import servo  # noqa
         except ImportError:
             subprocess.check_call(["pip", "install", "-r",
                                    path.join("tests", "wpt", "harness", "requirements.txt")])
@@ -311,7 +310,7 @@ class MachCommands(CommandBase):
         # before the virtualenv is initalised it doesn't see the blessings module so we don't
         # get coloured output. Setting the blessings global explicitly fixes that.
         from mozlog.structured.formatters import machformatter
-        import blessings
+        import blessings  # noqa
         machformatter.blessings = blessings
 
     def get_exec(self, name, default=None):

servo/python/tidy.py

@@ -7,17 +7,21 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-#!/usr/bin/env python
-
 import os
 import fnmatch
 import itertools
 import re
+import sys
 from licenseck import licenses
 
 filetypes_to_check = [".rs", ".rc", ".cpp", ".c", ".h", ".py"]
 reftest_directories = ["tests/ref"]
 reftest_filetype = ".list"
+python_dependencies = [
+    "./python/dependencies/flake8-2.4.1-py2.py3-none-any.whl",
+    "./python/dependencies/pep8-1.5.7-py2.py3-none-any.whl",
+    "./python/dependencies/pyflakes-0.9.0-py2.py3-none-any.whl",
+]
 
 ignored_files = [
     # Upstream
@@ -80,12 +84,14 @@ def check_length(idx, line):
     if len(line) >= 120:
         yield (idx + 1, "(much) overlong line")
 
+
 def check_whatwg_url(idx, line):
     match = re.search(r"https://html\.spec\.whatwg\.org/multipage/[\w-]+\.html#([\w\:-]+)", line)
     if match is not None:
         preferred_link = "https://html.spec.whatwg.org/multipage/#{}".format(match.group(1))
         yield (idx + 1, "link to WHATWG may break in the future, use this format instead: {}".format(preferred_link))
 
+
 def check_whitespace(idx, line):
     if line[-1] == "\n":
         line = line[:-1]
@@ -101,6 +107,7 @@ def check_whitespace(idx, line):
     if "\r" in line:
         yield (idx + 1, "CR on line")
 
+
 def check_by_line(contents):
     lines = contents.splitlines(True)
     for idx, line in enumerate(lines):
@@ -113,6 +120,25 @@ def check_by_line(contents):
             yield error
 
 
+def check_flake8(file_paths):
+    from flake8.main import check_file
+
+    ignore = {
+        "W291",  # trailing whitespace; the standard tidy process will enforce no trailing whitespace
+        "E501",  # 80 character line length; the standard tidy process will enforce line length
+    }
+
+    num_errors = 0
+
+    for file_path in file_paths:
+        if os.path.splitext(file_path)[-1].lower() != ".py":
+            continue
+
+        num_errors += check_file(file_path, ignore=ignore)
+
+    return num_errors
+
+
 def collect_errors_for_files(files_to_check, checking_functions):
     for file_name in files_to_check:
         with open(file_name, "r") as fp:
@@ -129,7 +155,7 @@ def check_reftest_order(files_to_check):
             split_lines = fp.read().splitlines()
             lines = filter(lambda l: len(l) > 0 and l[0] != '#', split_lines)
             for idx, line in enumerate(lines[:-1]):
-                next_line = lines[idx+1]
+                next_line = lines[idx + 1]
                 current = get_reftest_names(line)
                 next = get_reftest_names(next_line)
                 if current is not None and next is not None and current > next:
@@ -146,9 +172,13 @@ def get_reftest_names(line):
 
 
 def scan():
+    sys.path += python_dependencies
+
     all_files = collect_file_names()
     files_to_check = filter(should_check, all_files)
 
+    num_flake8_errors = check_flake8(files_to_check)
+
     checking_functions = [check_license, check_by_line]
     errors = collect_errors_for_files(files_to_check, checking_functions)
 
@@ -158,7 +188,7 @@ def scan():
 
     errors = list(itertools.chain(errors, r_errors))
 
-    if errors:
+    if errors or num_flake8_errors:
         for error in errors:
             print("{}:{}: {}".format(*error))
         return 1

servo/tests/jquery/run_jquery.py

@@ -22,24 +22,24 @@ import urlparse
 # See https://github.com/servo/servo/issues/6210 and
 #     https://github.com/servo/servo/issues/6211
 JQUERY_MODULES = [
-    #"ajax",            # panics
-    #"attributes",
-    #"callbacks",
-    #"core",            # mozjs crash
-    #"css",
-    #"data",
-    #"deferred",
-    #"dimensions",
-    #"effects",
-    #"event",           # panics
-    #"manipulation",    # mozjs crash
-    #"offset",
-    #"queue",
+    # "ajax",            # panics
+    # "attributes",
+    # "callbacks",
+    # "core",            # mozjs crash
+    # "css",
+    # "data",
+    # "deferred",
+    # "dimensions",
+    # "effects",
+    # "event",           # panics
+    # "manipulation",    # mozjs crash
+    # "offset",
+    # "queue",
     "selector",
-    #"serialize",
-    #"support",
-    #"traversing",
-    #"wrap"
+    # "serialize",
+    # "support",
+    # "traversing",
+    # "wrap"
 ]
 
 # Port to run the HTTP server on for jQuery.
@@ -48,6 +48,7 @@ TEST_SERVER_PORT = 8192
 # A regex for matching console.log output lines from the test runner.
 REGEX_PATTERN = "^\[jQuery test\] \[([0-9]+)/([0-9]+)/([0-9]+)] (.*)"
 
+
 # The result of a single test group.
 class TestResult:
     def __init__(self, success, fail, total, text):
@@ -94,7 +95,7 @@ def parse_string_to_results(buffer):
 # Run servo and print / parse the results for a specific jQuery test module.
 def run_servo(servo_exe, module):
     url = "http://localhost:{0}/jquery/test/?module={1}".format(TEST_SERVER_PORT, module)
-    args = [ servo_exe, url, "-z", "-f" ]
+    args = [servo_exe, url, "-z", "-f"]
     proc = subprocess.Popen(args, stdout=subprocess.PIPE)
     while True:
         line = proc.stdout.readline()
@@ -107,26 +108,31 @@ def run_servo(servo_exe, module):
         except AttributeError:
             pass
 
+
 # Build the filename for an expected results file.
 def module_filename(module):
     return 'expected_{0}.txt'.format(module)
 
+
 # Read an existing set of expected results to compare against.
 def read_existing_results(module):
     with open(module_filename(module), 'r') as file:
         buffer = file.read()
         return parse_string_to_results(buffer)
 
+
 # Write a set of results to file
 def write_results(module, results):
     with open(module_filename(module), 'w') as file:
         for result in test_results.itervalues():
             file.write(result.text + '\n')
 
+
 # Print usage if command line args are incorrect
 def print_usage():
     print("USAGE: {0} servo_binary jquery_base_dir test|update".format(sys.argv[0]))
 
+
 # Run a simple HTTP server to serve up the jQuery test suite
 def run_http_server():
     class ThreadingSimpleServer(SocketServer.ThreadingMixIn,

servo/tests/power/PowerMeasure.py

@@ -38,7 +38,7 @@ def PowerCollector(OutputDir, Benchmarks, LayoutThreads, Renderer):
     powerTiming = 1
     ExperimentNum = 21
     for ExpNum in range(1, ExperimentNum):
-        for layoutT in range(1, LayoutThreads+1):
+        for layoutT in range(1, LayoutThreads + 1):
             print("  layoutT=%d ExpNum=%d" % (layoutT, ExpNum))
             PowerFiles = path.join(
                 power_dir, "power-Layout%d-set%d.csv" % (layoutT, ExpNum))
@@ -71,7 +71,7 @@ def PowerParser(OutputDir, LayoutThreads):
     ResultFile.write("LayoutThreads, MeanPower, MaxPower , MinPower, MeanTime , MaxTime, "
                      "MinTime  \n")
 
-    for layoutT in range(1, LayoutThreads+1):
+    for layoutT in range(1, LayoutThreads + 1):
         MaxTime = 0
         MinTime = 1000000
         MaxPower = 0
@@ -132,8 +132,8 @@ def PowerParser(OutputDir, LayoutThreads):
             TimeGen = TimeGen + TotalTime
             PowerGen = PowerGen + TotalPower
 
-        TotalPower = PowerGen / float(ExperimentNum-1)
-        TotalTime = TimeGen / float(ExperimentNum-1)
+        TotalPower = PowerGen / float(ExperimentNum - 1)
+        TotalTime = TimeGen / float(ExperimentNum - 1)
         ResultFile.write(str(layoutT) + " , " + str(TotalPower) + " , " +
                          str(MaxPower) + " , " + str(MinPower) + " , " +
                          str(TotalTime) + " , " + str(MaxTime) + " , " +