bug 1442306 - generate a buildhub.json file during packaging. r=gps

buildhub is a service that stores a list of nightly and release builds and
can be queried to find specific builds. Currently it ingests data by scraping
info from ftp.mozilla.org. This patch makes it so we generate a buildhub.json
during packaging with the data that buildhub wants.

There are a few pieces of data that we can't accurately provide from the
build system such as the URL to the build, so we provide some stub data
there with the expectation that a release engineering process will fill
them in later.

MozReview-Commit-ID: 266BnZZBFoL

--HG--
extra : rebase_source : 23d05a9a0dc95ff3705551a5d85d90d6ed8f950f
extra : histedit_source : 166a8dbf4fb63de7e926fb292de07c550db96d78

toolkit/mozapps/installer/informulate.py

@@ -9,18 +9,24 @@
 from __future__ import unicode_literals
 
 from argparse import ArgumentParser
-import json
+import datetime
 import buildconfig
+import json
+import mozinfo
 import os
 
 
 def main():
     parser = ArgumentParser()
     parser.add_argument('output_json', help='Output JSON file')
+    parser.add_argument('buildhub_json', help='Output buildhub JSON file')
     parser.add_argument('output_txt', help='Output text file')
     # TODO: Move package-name.mk variables into moz.configure.
     parser.add_argument('pkg_platform', help='Package platform identifier')
+    parser.add_argument('--package', help='Path to application package file')
+    parser.add_argument('--installer', help='Path to application installer file')
     args = parser.parse_args()
+    mozinfo.find_and_update_from_json()
 
     important_substitutions = [
         'target_alias', 'target_cpu', 'target_os', 'target_vendor',
@@ -43,6 +49,50 @@ def main():
         json.dump(all_key_value_pairs, f, indent=2, sort_keys=True)
         f.write('\n')
 
+    with open(args.buildhub_json, 'wb') as f:
+        if args.installer and os.path.exists(args.installer):
+            package = args.installer
+        else:
+            package = args.package
+        build_time = datetime.datetime.strptime(build_id, '%Y%m%d%H%M%S')
+        st = os.stat(package)
+        mtime = datetime.datetime.fromtimestamp(st.st_mtime)
+        s = buildconfig.substs
+        record = {
+            'build': {
+                'id': build_id,
+                'date': build_time.isoformat() + 'Z',
+                'as': s['AS'],
+                'cc': s['CC'],
+                'cxx': s['CXX'],
+                'host': s['host_alias'],
+                'target': s['target_alias'],
+            },
+            'source': {
+                'product': s['MOZ_APP_NAME'],
+                'repository': s['MOZ_SOURCE_REPO'],
+                'tree': os.environ['MH_BRANCH'],
+                'revision': s['MOZ_SOURCE_CHANGESET'],
+            },
+            'target': {
+                'platform': args.pkg_platform,
+                'os': mozinfo.info['os'],
+                # This would be easier if the locale was specified at configure time.
+                'locale': os.environ.get('AB_CD', 'en-US'),
+                'version': s['MOZ_APP_VERSION'],
+                'channel': s['MOZ_UPDATE_CHANNEL'],
+            },
+            'download': {
+                # The release pipeline will update these keys.
+                'url': os.path.basename(package),
+                'mimetype': 'application/octet-stream',
+                'date': mtime.isoformat() + 'Z',
+                'size': st.st_size,
+            }
+        }
+        json.dump(record, f, indent=2, sort_keys=True)
+        f.write('\n')
+
     with open(args.output_txt, 'wb') as f:
         f.write('buildID={}\n'.format(build_id))
 

toolkit/mozapps/installer/package-name.mk

@@ -112,6 +112,7 @@ endif
 
 MOZ_SOURCESTAMP_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).txt
 MOZ_BUILDINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).json
+MOZ_BUILDHUB_JSON = $(DIST)/$(PKG_PATH)/buildhub.json
 MOZ_BUILDID_INFO_TXT_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME)_info.txt
 MOZ_MOZINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).mozinfo.json
 MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/$(PKG_BASENAME).test_packages.json

toolkit/mozapps/installer/packager.mk

@@ -94,7 +94,10 @@ endif
 ifdef MOZ_AUTOMATION
 	cp $(DEPTH)/mozinfo.json $(MOZ_MOZINFO_FILE)
 	$(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/informulate.py \
-		$(MOZ_BUILDINFO_FILE) $(MOZ_BUILDID_INFO_TXT_FILE) $(MOZ_PKG_PLATFORM)
+		$(MOZ_BUILDINFO_FILE) $(MOZ_BUILDHUB_JSON) $(MOZ_BUILDID_INFO_TXT_FILE) \
+		$(MOZ_PKG_PLATFORM) \
+		--package=$(DIST)/$(PACKAGE) \
+		--installer=$(INSTALLER_PACKAGE)
 endif
 	$(TOUCH) $@
 

toolkit/mozapps/installer/upload-files.mk

@@ -394,6 +394,7 @@ UPLOAD_FILES= \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(MOZ_SOURCESTAMP_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDINFO_FILE)) \
+  $(call QUOTED_WILDCARD,$(MOZ_BUILDHUB_JSON)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDID_INFO_TXT_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \