Bug 977071 - Add run_info to structured log suite_start message and fix some small bugs. r=ahal

testing/mozbase/mozlog/mozlog/__init__.py

@@ -9,8 +9,7 @@ It simply wraps Python's logging_ module and adds a few convenience methods
 for logging test results and events.
 
 The structured submodule takes a different approach and implements a
-JSON-based logging protocol designed for recording test results.
-"""
+JSON-based logging protocol designed for recording test results."""
 
 from logger import *
 from loglistener import LogMessageServer
@@ -22,3 +21,4 @@ except ImportError:
     # Structured logging doesn't work on python 2.6 which is still used on some
     # legacy test machines; https://bugzilla.mozilla.org/show_bug.cgi?id=864866
     pass
+

testing/mozbase/mozlog/mozlog/structured/commandline.py

@@ -15,6 +15,7 @@ log_formatters = {
     'xunit': (formatters.XUnitFormatter, "xUnit compatible XML"),
     'html': (formatters.HTMLFormatter, "HTML report"),
     'mach': (formatters.MachFormatter, "Uncolored mach-like output"),
+    'mach_terminal': (formatters.MachTerminalFormatter, "Colored mach-like output for use in a tty"),
 }
 
 
@@ -42,7 +43,7 @@ def add_logging_group(parser):
                                       "and takes a filename to write that format to, "
                                       "or '-' to write to stdout.")
     for name, (cls, help_str) in log_formatters.iteritems():
-        group.add_argument("--log-" + name, type=log_file,
+        group.add_argument("--log-" + name, action="append", type=log_file,
                            help=help_str)
 
 
@@ -51,8 +52,8 @@ def setup_logging(suite, args, defaults):
     Configure a structuredlogger based on command line arguments.
 
     :param suite: The name of the testsuite being run
-    :param args: The Namespace object produced by argparse from parsing
-                 command line arguments from a parser with logging arguments.
+    :param args: A dictionary of {argument_name:value} produced from
+                 parsing the command line arguments for the application
     :param defaults: A dictionary of {formatter name: output stream} to apply
                      when there is no logging supplied on the command line.
 
@@ -62,14 +63,15 @@ def setup_logging(suite, args, defaults):
     prefix = "log_"
     found = False
     found_stdout_logger = False
-    for name, value in args.iteritems():
-        if name.startswith(prefix) and value is not None:
-            found = True
-            if value == sys.stdout:
-                found_stdout_logger = True
-            formatter_cls = log_formatters[name[len(prefix):]][0]
-            logger.add_handler(handlers.StreamHandler(stream=value,
-                                                      formatter=formatter_cls()))
+    for name, values in args.iteritems():
+        if name.startswith(prefix) and values is not None:
+            for value in values:
+                found = True
+                if value == sys.stdout:
+                    found_stdout_logger = True
+                formatter_cls = log_formatters[name[len(prefix):]][0]
+                logger.add_handler(handlers.StreamHandler(stream=value,
+                                                          formatter=formatter_cls()))
 
     #If there is no user-specified logging, go with the default options
     if not found:

testing/mozbase/mozlog/mozlog/structured/formatters/__init__.py

@@ -8,4 +8,5 @@ from xunit import XUnitFormatter
 from html import HTMLFormatter
 from machformatter import MachFormatter, MachTerminalFormatter
 
-JSONFormatter = lambda: json.dumps
+def JSONFormatter():
+    return lambda x: json.dumps(x) + "\n"

testing/mozbase/mozlog/mozlog/structured/formatters/html/html.py

@@ -4,6 +4,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
+import json
 import datetime
 import os
 

testing/mozbase/mozlog/mozlog/structured/formatters/machformatter.py

@@ -29,7 +29,7 @@ class BaseMachFormatter(base.BaseFormatter):
             return "%s %s\n" % (self.generic_formatter(data), s)
 
     def _get_test_id(self, data):
-        test_id = data["test"]
+        test_id = data.get("test")
         if isinstance(test_id, list):
             test_id = tuple(test_id)
         return test_id
@@ -62,12 +62,13 @@ class BaseMachFormatter(base.BaseFormatter):
     def test_status(self, data):
         test = self._get_test_id(data)
         if test not in self.status_buffer:
-            self.buffer[test] = {"count": 0, "unexpected": 0, "pass": 0}
-        self.buffer[test]["count"] += 1
+            self.status_buffer[test] = {"count": 0, "unexpected": 0, "pass": 0}
+        self.status_buffer[test]["count"] += 1
+
         if "expected" in data:
-            self.buffer[test]["unexpected"] += 1
+            self.status_buffer[test]["unexpected"] += 1
         if data["status"] == "PASS":
-            self.buffer[test]["pass"] += 1
+            self.status_buffer[test]["pass"] += 1
 
     def process_output(self, data):
         return '"%s" (pid:%s command:%s)' % (data["data"],
@@ -132,9 +133,9 @@ class MachTerminalFormatter(BaseMachFormatter):
     def __call__(self, data):
         s = BaseMachFormatter.__call__(self, data)
         if s is not None:
-            t = self.terminal.blue(format_seconds(self._time(entry)))
+            t = self.terminal.blue(format_seconds(self._time(data)))
 
-            return '%s %s' % (t, self._colorize(entry, s))
+            return '%s %s' % (t, self._colorize(data, s))
 
     def _colorize(self, data, s):
         if self.terminal is None:
@@ -155,6 +156,8 @@ class MachTerminalFormatter(BaseMachFormatter):
 
         if color is not None:
             result = color(s[:len_action]) + s[len_action:]
+        else:
+            result = s
 
         return result
 

testing/mozbase/mozlog/mozlog/structured/handlers/__init__.py

@@ -4,6 +4,8 @@
 
 from threading import Lock
 
+from ..structuredlog import log_levels
+
 
 class BaseHandler(object):
     def __init__(self, formatter=str):

testing/mozbase/mozlog/mozlog/structured/structuredlog.py

@@ -104,12 +104,15 @@ class StructuredLogger(object):
         all_data.update(data)
         return all_data
 
-    def suite_start(self, tests):
+    def suite_start(self, tests, run_info=None):
         """Log a suite_start message
 
         :param tests: List of test identifiers that will be run in the suite.
         """
-        self._log_data("suite_start", {"tests": tests})
+        data = {"tests": tests}
+        if run_info is not None:
+            data["run_info"] = run_info
+        self._log_data("suite_start", data)
 
     def suite_end(self):
         """Log a suite_end message"""
@@ -171,8 +174,7 @@ class StructuredLogger(object):
         self._log_data("test_end", data)
 
     def process_output(self, process, data, command=None):
-        """
-        Log output from a managed process.
+        """Log output from a managed process.
 
         :param process: A unique identifier for the process producing the output
                         (typically the pid)