Bug 1415614 - Add an API to log all structured messages; r=mshal

Currently, marking a logger as a structured logger will require a
subsequent function call in order for the logger to be hooked up
to active handlers. This behavior is not intuitive and makes it
easy to not have handlers for newly-registered loggers. This means
messages may not be logged anywhere.

In addition, we have to manually specify which named loggers to
enable structured logging for. This can be annoying.

We change the behavior of register_structured_logger() to
automatically add existing terminal and json handlers to the
logger being marked as structured.

We also introduce an API to enable structured logging for all
loggers. Existing consumers of registered_structured_logger()
in mozbuild have been updated to use this API. A new consumer
has been added for the `mach configure` command because it should
have been there before.

We stop short of making enable_all_structured_loggers() the default.
This is because various commands interact with the log manager in
ways that will result in duplicate logging of messages and
dropping of structured messages. There is a bit of a rabbit hole
here and addressing it can be done as a follow-up.

MozReview-Commit-ID: 1aU6eJvTSMP

--HG--
extra : rebase_source : 2a0a569b378cc3083b55fc7076b291abdfb7453f

python/mach/mach/logging.py

@@ -256,10 +256,43 @@ class LoggingManager(object):
             self.terminal_handler.removeFilter(self.structured_filter)
             self.root_logger.removeHandler(self.terminal_handler)
 
-    def register_structured_logger(self, logger):
+    def register_structured_logger(self, logger, terminal=True, json=True):
         """Register a structured logger.
 
         This needs to be called for all structured loggers that don't chain up
         to the mach logger in order for their output to be captured.
         """
         self.structured_loggers.append(logger)
+
+        if terminal and self.terminal_handler:
+            logger.addHandler(self.terminal_handler)
+
+        if json:
+            for handler in self.json_handlers:
+                logger.addHandler(handler)
+
+    def enable_all_structured_loggers(self, terminal=True, json=True):
+        """Enable logging of all structured messages from all loggers.
+
+        ``terminal`` and ``json`` determine which log handlers to operate
+        on. By default, all known handlers are operated on.
+        """
+        # Remove current handlers from all loggers so we don't double
+        # register handlers.
+        for logger in self.root_logger.manager.loggerDict.values():
+            # Some entries might be logging.PlaceHolder.
+            if not isinstance(logger, logging.Logger):
+                continue
+
+            if terminal:
+                logger.removeHandler(self.terminal_handler)
+
+            if json:
+                for handler in self.json_handlers:
+                    logger.removeHandler(handler)
+
+        # Wipe out existing registered structured loggers since they
+        # all propagate to root logger.
+        self.structured_loggers = []
+        self.register_structured_logger(self.root_logger, terminal=terminal,
+                                        json=json)

python/mozbuild/mozbuild/mach_commands.py

@@ -394,7 +394,7 @@ class Build(MachCommandBase):
             resolve_target_to_make,
         )
 
-        self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
+        self.log_manager.enable_all_structured_loggers()
 
         warnings_path = self._get_state_filename('warnings.json')
         monitor = self._spawn(BuildMonitor)
@@ -690,6 +690,8 @@ class Build(MachCommandBase):
     @CommandArgument('options', default=None, nargs=argparse.REMAINDER,
                      help='Configure options')
     def configure(self, options=None, buildstatus_messages=False, line_handler=None):
+        self.log_manager.enable_all_structured_loggers()
+
         def on_line(line):
             self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
 
@@ -2214,6 +2216,8 @@ class StaticAnalysis(MachCommandBase):
     def check(self, source=None, jobs=2, strip=1, verbose=False,
               checks='-*', fix=False, header_filter=''):
         self._set_log_level(verbose)
+        self.log_manager.enable_all_structured_loggers()
+
         rc = self._build_compile_db(verbose=verbose)
         if rc != 0:
             return rc
@@ -2246,8 +2250,6 @@ class StaticAnalysis(MachCommandBase):
         if fix:
             common_args.append('-fix')
 
-        self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
-
         compile_db = json.loads(open(self._compile_db, 'r').read())
         total = 0
         import re