Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-08 19:04:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1198179 - Kill gen_mach_buildprops.py; r=ted

Browse Source
This commit is contained in:
Mike Shal 2015-08-25 11:42:44 -04:00
parent eb99962783
commit 66d8bc270e
5 changed files with 39 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
build/gen_mach_buildprops.py
View File

@ -1,81 +0,0 @@
#!/usr/bin/python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import sys
import os
import hashlib
import json
import re
import errno
from argparse import ArgumentParser
def getFileHashAndSize(filename):
sha512Hash = 'UNKNOWN'
size = 'UNKNOWN'
try:
# open in binary mode to make sure we get consistent results
# across all platforms
f = open(filename, "rb")
shaObj = hashlib.sha512(f.read())
sha512Hash = shaObj.hexdigest()
size = os.path.getsize(filename)
except:
pass
return (sha512Hash, size)
def getMarProperties(filename, partial=False):
if not os.path.exists(filename):
return {}
(mar_hash, mar_size) = getFileHashAndSize(filename)
martype = 'partial' if partial else 'complete'
return {
'%sMarFilename' % martype: os.path.basename(filename),
'%sMarSize' % martype: mar_size,
'%sMarHash' % martype: mar_hash,
}
def getPartialInfo(props):
return [{
"from_buildid": props.get("previous_buildid"),
"size": props.get("partialMarSize"),
"hash": props.get("partialMarHash"),
"url": props.get("partialMarUrl"),
}]
if __name__ == '__main__':
parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
parser.add_argument("--complete-mar-file", required=True,
action="store", dest="complete_mar_file",
help="Path to the complete MAR file, relative to the objdir.")
parser.add_argument("--partial-mar-file", required=False,
action="store", dest="partial_mar_file",
help="Path to the partial MAR file, relative to the objdir.")
parser.add_argument("--upload-properties", required=False,
action="store", dest="upload_properties",
help="Path to the properties written by 'make upload'")
args = parser.parse_args()
json_data = getMarProperties(args.complete_mar_file)
if args.upload_properties:
with open(args.upload_properties) as f:
json_data.update(json.load(f))
if args.partial_mar_file:
json_data.update(getMarProperties(args.partial_mar_file, partial=True))
# Pull the previous buildid from the partial mar filename.
res = re.match(r'.*\.([0-9]+)-[0-9]+.mar', args.partial_mar_file)
if res:
json_data['previous_buildid'] = res.group(1)
# Set partialInfo to be a collection of the partial mar properties
# useful for balrog.
json_data['partialInfo'] = getPartialInfo(json_data)
with open('mach_build_properties.json', 'w') as outfile:
json.dump(json_data, outfile, indent=4)

7
build/moz-automation.mk
View File

@ -17,11 +17,6 @@ include $(topsrcdir)/toolkit/mozapps/installer/upload-files.mk
# Clear out DIST_FILES if it was set by upload-files.mk (for Android builds)
DIST_FILES =
ifeq (1,$(MOZ_AUTOMATION_UPLOAD))
# Properties from 'make upload' that are file URLs.
AUTOMATION_UPLOAD_PROPERTIES = --upload-properties $(DIST)/upload-properties.json
endif
# Helper variables to convert from MOZ_AUTOMATION_* variables to the
# corresponding the make target
tier_MOZ_AUTOMATION_BUILD_SYMBOLS = buildsymbols
@ -100,7 +95,7 @@ automation/l10n-check: automation/pretty-l10n-check
automation/update-packaging: automation/pretty-update-packaging
automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) $(addprefix --partial-mar-file ,$(wildcard $(DIST)/$(PARTIAL_MAR))) $(AUTOMATION_UPLOAD_PROPERTIES)
@echo Automation steps completed.
# Note: We have to force -j1 here, at least until bug 1036563 is fixed.
AUTOMATION_EXTRA_CMDLINE-l10n-check = -j1

35
build/upload.py
View File

@ -25,6 +25,8 @@
import sys, os
import re
import json
import errno
import hashlib
from optparse import OptionParser
from subprocess import check_call, check_output, STDOUT
import redo
@ -119,6 +121,33 @@ def GetRemotePath(path, local_file, base_path):
dir = dir[len(base_path)+1:].replace('\\','/')
return path + dir
def GetFileHashAndSize(filename):
sha512Hash = 'UNKNOWN'
size = 'UNKNOWN'
try:
# open in binary mode to make sure we get consistent results
# across all platforms
with open(filename, "rb") as f:
shaObj = hashlib.sha512(f.read())
sha512Hash = shaObj.hexdigest()
size = os.path.getsize(filename)
except:
raise Exception("Unable to get filesize/hash from file: %s" % filename)
return (sha512Hash, size)
def GetMarProperties(filename):
if not os.path.exists(filename):
return {}
(mar_hash, mar_size) = GetFileHashAndSize(filename)
return {
'completeMarFilename': os.path.basename(filename),
'completeMarSize': mar_size,
'completeMarHash': mar_hash,
}
def GetUrlProperties(output, package):
# let's create a switch case using name-spaces/dict
# rather than a long if/else with duplicate code
@ -174,6 +203,7 @@ def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None,
if base_path is not None:
base_path = os.path.abspath(base_path)
remote_files = []
properties = {}
try:
for file in files:
file = os.path.abspath(file)
@ -186,6 +216,9 @@ def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None,
print "Uploading " + file
DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
remote_files.append(remote_path + '/' + os.path.basename(file))
if file.endswith('.complete.mar'):
properties.update(GetMarProperties(file))
if post_upload_command is not None:
if verbose:
print "Running post-upload command: " + post_upload_command
@ -195,7 +228,7 @@ def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None,
print output
if properties_file:
with open(properties_file, 'w') as outfile:
properties = GetUrlProperties(output, package)
properties.update(GetUrlProperties(output, package))
properties['packageFilename'] = package
properties['uploadFiles'] = [os.path.abspath(f) for f in files]
json.dump(properties, outfile, indent=4)

8
testing/mozharness/mozharness/mozilla/building/buildbase.py
View File

@ -1241,7 +1241,7 @@ or run without that action (ie: --no-{action})"
def _query_props_set_by_mach(self, console_output=True, error_level=FATAL):
mach_properties_path = os.path.join(
self.query_abs_dirs()['abs_obj_dir'], 'mach_build_properties.json'
self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json'
)
self.info("setting properties set by mach build. Looking in path: %s"
% mach_properties_path)
@ -1262,9 +1262,7 @@ or run without that action (ie: --no-{action})"
if prop != 'UNKNOWN':
self.set_buildbot_property(key, prop, write_to_file=True)
else:
self.log("Could not determine path for build properties. "
"Does this exist: `%s` ?" % mach_properties_path,
level=error_level)
self.info("No mach_build_properties.json found - not importing properties.")
def generate_build_props(self, console_output=True, halt_on_failure=False):
"""sets props found from mach build and, in addition, buildid,
@ -1644,7 +1642,7 @@ or run without that action (ie: --no-{action})"
self._run_tooltool()
self._create_mozbuild_dir()
mach_props = os.path.join(
self.query_abs_dirs()['abs_obj_dir'], 'mach_build_properties.json'
self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json'
)
if os.path.exists(mach_props):
self.info("Removing previous mach property file: %s" % mach_props)

2
toolkit/mozapps/installer/packager.mk
View File

@ -209,7 +209,7 @@ checksum:
upload: checksum
$(PYTHON) $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) \
--package $(PACKAGE) \
--properties-file $(DIST)/upload-properties.json \
--properties-file $(DIST)/mach_build_properties.json \
$(UPLOAD_FILES) \
$(CHECKSUM_FILES)