Bug 1066323 - [mozlog] Mozlog shouldn't propagate unremarkable defaults to the raw log r=jgraham

Currently, some of the raw JSON logs for mochitest and marionette, et al, include
empty dictionaries, None values and other unremarkable values that are marked
as optional. This fix aims to remove these unnecessary items from being
passed to the raw log.

A method has been added to the log_actions class which removes defaults if they
are marked as optional and the value is included in the default list. This is
called on the kwargs returned by the convert_known method, before being
propagated to the log_raw method for StructuredLogger.

Differential Revision: https://phabricator.services.mozilla.com/D25081

--HG--
extra : moz-landing-system : lando

testing/mozbase/mozlog/mozlog/logtypes.py

@@ -21,6 +21,7 @@ class log_action(object):
 
         self.args_no_default = []
         self.args_with_default = []
+        self.optional_args = set()
 
         # These are the required fields in a log message that usually aren't
         # supplied by the caller, but can be in the case of log_raw
@@ -38,6 +39,9 @@ class log_action(object):
             else:
                 self.args_with_default.append(arg.name)
 
+            if arg.optional:
+                self.optional_args.add(arg.name)
+
             if arg.name in self.args:
                 raise ValueError("Repeated argument name %s" % arg.name)
 
@@ -94,7 +98,10 @@ class log_action(object):
             if key in self.args:
                 out_value = self.args[key](value)
                 if out_value is not missing:
-                    data[key] = out_value
+                    if (key in self.optional_args and value == self.args[key].default):
+                        pass
+                    else:
+                        data[key] = out_value
             else:
                 raise TypeError("Unrecognised argument %s" % key)
 
@@ -106,6 +113,8 @@ class log_action(object):
         return self.convert(**known_kwargs)
 
 
+
+
 class DataType(object):
 
     def __init__(self, name, default=no_default, optional=False):

testing/mozbase/mozlog/mozlog/structuredlog.py

@@ -230,7 +230,8 @@ class StructuredLogger(object):
         action = raw_data["action"]
         converted_data = convertor_registry[action].convert_known(**raw_data)
         for k, v in six.iteritems(raw_data):
-            if k not in converted_data:
+            if (k not in converted_data and
+                    k not in convertor_registry[action].optional_args):
                 converted_data[k] = v
 
         data = self._make_log_data(action, converted_data)

testing/mozbase/mozlog/tests/test_structured.py

@@ -257,6 +257,26 @@ class TestStructuredLog(BaseStructuredTest):
         self.assertTrue(self.pop_last_item()["message"].startswith(
             "test_status for test_UNKNOWN logged while not in progress. Logged with data: {"))
 
+    def test_remove_optional_defaults(self):
+        self.logger.suite_start([])
+        self.logger.test_start("test1")
+        self.logger.test_status("test1", "subtest name", "fail", message=None, stack=None)
+        self.assert_log_equals({"action": "test_status",
+                                "subtest": "subtest name",
+                                "status": "FAIL",
+                                "expected": "PASS",
+                                "test": "test1"})
+        self.logger.test_end("test1", "OK")
+        self.logger.suite_end()
+
+    def test_remove_optional_defaults_raw_log(self):
+        self.logger.log_raw({"action": "suite_start",
+                             "tests": [1],
+                             "name": None})
+        self.assert_log_equals({"action": "suite_start",
+                                "tests": {"default": ["1"]}})
+        self.logger.suite_end()
+
     def test_end(self):
         self.logger.suite_start([])
         self.logger.test_start("test1")