Bug 1306078 - part 3 - make mach vendor rust check for overly-large files; r=ted.mielczarek

When vendoring third-party files, we'd like an explicit notice/review
when said files contain a "large file".  This commit adds such checks
for files vendored via `mach vendor rust`.

As we don't yet have a server-side hook in place to prevent large files
from being added, we just have a command-line flag that people are
expected to use, on the honor system, to permit large files to be added
when vendoring.

python/mozbuild/mozbuild/mach_commands.py

@@ -1594,6 +1594,9 @@ class Vendor(MachCommandBase):
     @CommandArgument('--ignore-modified', action='store_true',
         help='Ignore modified files in current checkout',
         default=False)
+    @CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true',
+        help='Permit overly-large files to be added to the repository',
+        default=False)
     def vendor_rust(self, **kwargs):
         from mozbuild.vendor_rust import VendorRust
         vendor_command = self._spawn(VendorRust)

python/mozbuild/mozbuild/vendor_rust.py

@@ -80,7 +80,8 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore-
         self.log(logging.ERROR, 'openssl', {}, "OpenSSL not found!")
         return None
 
-    def vendor(self, ignore_modified=False):
+    def vendor(self, ignore_modified=False,
+               build_peers_said_large_imports_were_ok=False):
         self.populate_logger()
         self.log_manager.enable_unstructured()
         if not ignore_modified:
@@ -99,7 +100,8 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore-
                              append_env=env)
         else:
             self.log(logging.DEBUG, 'cargo_vendor', {}, 'cargo-vendor already intalled')
-        vendor_dir = mozpath.join(self.topsrcdir, 'third_party/rust')
+        relative_vendor_dir = 'third_party/rust'
+        vendor_dir = mozpath.join(self.topsrcdir, relative_vendor_dir)
         self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % vendor_dir)
         mozfile.remove(vendor_dir)
         # Once we require a new enough cargo to switch to workspaces, we can
@@ -114,6 +116,41 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore-
             # We do an |update -p| here to regenerate the Cargo.lock file with minimal changes. See bug 1324462
             self._run_command_in_srcdir(args=[cargo, 'update', '--manifest-path', mozpath.join(path, 'Cargo.toml'), '-p', lib])
             self._run_command_in_srcdir(args=[cargo, 'vendor', '--sync', mozpath.join(path, 'Cargo.lock'), vendor_dir])
-        #TODO: print stats on size of files added/removed, warn or error
-        # when adding very large files (bug 1306078)
         self.repository.add_remove_files(vendor_dir)
+
+        large_files = set()
+        cumulative_added_size = 0
+        for f in self.repository.get_added_files():
+            path = mozpath.join(self.topsrcdir, f)
+            size = os.stat(path).st_size
+            cumulative_added_size += size
+            if size > FILESIZE_LIMIT:
+                large_files.add(f)
+
+        # Forcefully complain about large files being added, as history has
+        # shown that large-ish files typically are not needed.
+        if large_files and not build_peers_said_large_imports_were_ok:
+            self.log(logging.ERROR, 'filesize_check', {},
+                     '''The following files exceed the filesize limit of {size}:
+
+{files}
+
+Please find a way to reduce the sizes of these files or talk to a build
+peer about the particular large files you are adding.
+
+The changes from `mach vendor rust` will NOT be added to version control.
+'''.format(files='\n'.join(sorted(large_files)), size=FILESIZE_LIMIT))
+            self.repository.forget_add_remove_files()
+            sys.exit(1)
+
+        # Only warn for large imports, since we may just have large code
+        # drops from time to time (e.g. importing features into m-c).
+        SIZE_WARN_THRESHOLD = 5 * 1024 * 1024
+        if cumulative_added_size >= SIZE_WARN_THRESHOLD:
+            self.log(logging.WARN, 'filesize_check', {},
+                     '''Your changes add {size} bytes of added files.
+
+Please consider finding ways to reduce the size of the vendored packages.
+For instance, check the vendored packages for unusually large test or
+benchmark files that don't need to be published to crates.io and submit
+a pull request upstream to ignore those files when publishing.'''.format(size=cumulative_added_size))