mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-12 12:55:46 +00:00
Bug 1194709 - Add pushid support for mozharness routes; r=jlund
This commit is contained in:
Mike Shal
parent
6ce32f63b0
commit
e4ec003335
@ -1,10 +1,12 @@
|
||||
import os
|
||||
import re
|
||||
import urlparse
|
||||
from collections import namedtuple
|
||||
|
||||
from mozharness.base.script import ScriptMixin
|
||||
from mozharness.base.log import LogMixin, OutputParser, WARNING
|
||||
from mozharness.base.errors import HgErrorList, VCSException
|
||||
from mozharness.base.transfer import TransferMixin
|
||||
|
||||
HgtoolErrorList = [{
|
||||
'substr': 'abort: HTTP Error 404: Not Found',
|
||||
@ -28,7 +30,7 @@ class HgtoolParser(OutputParser):
|
||||
super(HgtoolParser, self).parse_single_line(line)
|
||||
|
||||
|
||||
class HgtoolVCS(ScriptMixin, LogMixin):
|
||||
class HgtoolVCS(ScriptMixin, LogMixin, TransferMixin):
|
||||
def __init__(self, log_obj=None, config=None, vcs_config=None,
|
||||
script_obj=None):
|
||||
super(HgtoolVCS, self).__init__()
|
||||
@ -113,3 +115,38 @@ class HgtoolVCS(ScriptMixin, LogMixin):
|
||||
raise VCSException("Unable to checkout")
|
||||
|
||||
return parser.got_revision
|
||||
|
||||
def query_pushinfo(self, repository, revision):
|
||||
"""Query the pushdate and pushid of a repository/revision.
|
||||
This is intended to be used on hg.mozilla.org/mozilla-central and
|
||||
similar. It may or may not work for other hg repositories.
|
||||
"""
|
||||
PushInfo = namedtuple('PushInfo', ['pushid', 'pushdate'])
|
||||
|
||||
try:
|
||||
url = '%s/json-pushes?changeset=%s' % (repository, revision)
|
||||
self.info('Pushdate URL is: %s' % url)
|
||||
contents = self.retry(self.load_json_from_url, args=(url,))
|
||||
|
||||
# The contents should be something like:
|
||||
# {
|
||||
# "28537": {
|
||||
# "changesets": [
|
||||
# "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5",
|
||||
# ],
|
||||
# "date": 1428072488,
|
||||
# "user": "user@mozilla.com"
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# So we grab the first element ("28537" in this case) and then pull
|
||||
# out the 'date' field.
|
||||
pushid = contents.iterkeys().next()
|
||||
self.info('Pushid is: %s' % pushid)
|
||||
pushdate = contents[pushid]['date']
|
||||
self.info('Pushdate is: %s' % pushdate)
|
||||
return PushInfo(pushid, pushdate)
|
||||
|
||||
except Exception:
|
||||
self.exception("Failed to get push info from hg.mozilla.org")
|
||||
raise
|
||||
|
||||
@ -53,19 +53,16 @@ class VCSMixin(object):
|
||||
self.rmtree(dest)
|
||||
raise
|
||||
|
||||
def _get_vcs_class(self, vcs):
|
||||
vcs = vcs or self.config.get('default_vcs', getattr(self, 'default_vcs', None))
|
||||
vcs_class = VCS_DICT.get(vcs)
|
||||
return vcs_class
|
||||
|
||||
def vcs_checkout(self, vcs=None, error_level=FATAL, **kwargs):
|
||||
""" Check out a single repo.
|
||||
"""
|
||||
c = self.config
|
||||
if not vcs:
|
||||
if c.get('default_vcs'):
|
||||
vcs = c['default_vcs']
|
||||
else:
|
||||
try:
|
||||
vcs = self.default_vcs
|
||||
except AttributeError:
|
||||
pass
|
||||
vcs_class = VCS_DICT.get(vcs)
|
||||
vcs_class = self._get_vcs_class(vcs)
|
||||
if not vcs_class:
|
||||
self.error("Running vcs_checkout with kwargs %s" % str(kwargs))
|
||||
raise VCSException("No VCS set!")
|
||||
@ -110,6 +107,20 @@ class VCSMixin(object):
|
||||
self.chdir(orig_dir)
|
||||
return revision_dict
|
||||
|
||||
def vcs_query_pushinfo(self, repository, revision, vcs=None):
|
||||
"""Query the pushid/pushdate of a repository/revision
|
||||
Returns a namedtuple with "pushid" and "pushdate" elements
|
||||
"""
|
||||
vcs_class = self._get_vcs_class(vcs)
|
||||
if not vcs_class:
|
||||
raise VCSException("No VCS set in vcs_query_pushinfo!")
|
||||
vcs_obj = vcs_class(
|
||||
log_obj=self.log_obj,
|
||||
config=self.config,
|
||||
script_obj=self,
|
||||
)
|
||||
return vcs_obj.query_pushinfo(repository, revision)
|
||||
|
||||
|
||||
class VCSScript(VCSMixin, BaseScript):
|
||||
def __init__(self, **kwargs):
|
||||
|
||||
@ -29,7 +29,6 @@ import re
|
||||
from mozharness.base.config import BaseConfig, parse_config_file
|
||||
from mozharness.base.log import ERROR, OutputParser, FATAL
|
||||
from mozharness.base.script import PostScriptRun
|
||||
from mozharness.base.transfer import TransferMixin
|
||||
from mozharness.base.vcs.vcsbase import MercurialScript
|
||||
from mozharness.mozilla.buildbot import BuildbotMixin, TBPL_STATUS_DICT, \
|
||||
TBPL_EXCEPTION, TBPL_RETRY, EXIT_STATUS_DICT, TBPL_WARNING, TBPL_SUCCESS, \
|
||||
@ -534,7 +533,7 @@ def generate_build_UID():
|
||||
|
||||
class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
|
||||
SigningMixin, VirtualenvMixin, MercurialScript,
|
||||
TransferMixin, InfluxRecordingMixin):
|
||||
InfluxRecordingMixin):
|
||||
def __init__(self, **kwargs):
|
||||
# objdir is referenced in _query_abs_dirs() so let's make sure we
|
||||
# have that attribute before calling BaseScript.__init__
|
||||
@ -559,7 +558,6 @@ class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
|
||||
self.repo_path = None
|
||||
self.buildid = None
|
||||
self.builduid = None
|
||||
self.pushdate = None
|
||||
self.query_buildid() # sets self.buildid
|
||||
self.query_builduid() # sets self.builduid
|
||||
self.generated_build_props = False
|
||||
@ -735,39 +733,6 @@ or run without that action (ie: --no-{action})"
|
||||
self.buildid = buildid
|
||||
return self.buildid
|
||||
|
||||
def query_pushdate(self):
|
||||
if self.pushdate:
|
||||
return self.pushdate
|
||||
|
||||
try:
|
||||
url = '%s/json-pushes?changeset=%s' % (
|
||||
self._query_repo(),
|
||||
self.query_revision(),
|
||||
)
|
||||
self.info('Pushdate URL is: %s' % url)
|
||||
contents = self.retry(self.load_json_from_url, args=(url,))
|
||||
|
||||
# The contents should be something like:
|
||||
# {
|
||||
# "28537": {
|
||||
# "changesets": [
|
||||
# "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5",
|
||||
# ],
|
||||
# "date": 1428072488,
|
||||
# "user": "user@mozilla.com"
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# So we grab the first element ("28537" in this case) and then pull
|
||||
# out the 'date' field.
|
||||
self.pushdate = contents.itervalues().next()['date']
|
||||
self.info('Pushdate is: %s' % self.pushdate)
|
||||
except Exception:
|
||||
self.exception("Failed to get pushdate from hg.mozilla.org")
|
||||
raise
|
||||
|
||||
return self.pushdate
|
||||
|
||||
def _query_objdir(self):
|
||||
if self.objdir:
|
||||
return self.objdir
|
||||
@ -1380,6 +1345,10 @@ or run without that action (ie: --no-{action})"
|
||||
self.warning('Skipping S3 file upload: No taskcluster credentials.')
|
||||
return
|
||||
|
||||
repo = self._query_repo()
|
||||
revision = self.query_revision()
|
||||
pushinfo = self.vcs_query_pushinfo(repo, revision)
|
||||
|
||||
# We need to create & activate the virtualenv so that we can import
|
||||
# taskcluster (and its dependent modules, like requests and hawk).
|
||||
# Normally we could create the virtualenv as an action, but due to some
|
||||
@ -1411,7 +1380,7 @@ or run without that action (ie: --no-{action})"
|
||||
fmt = {
|
||||
'index': index,
|
||||
'project': self.buildbot_config['properties']['branch'],
|
||||
'head_rev': self.query_revision(),
|
||||
'head_rev': revision,
|
||||
'build_product': self.config['stage_product'],
|
||||
'build_name': self.query_build_name(),
|
||||
'build_type': self.query_build_type(),
|
||||
@ -1422,7 +1391,7 @@ or run without that action (ie: --no-{action})"
|
||||
self.info("Using routes: %s" % routes)
|
||||
|
||||
tc = Taskcluster(self.branch,
|
||||
self.query_pushdate(), # Use pushdate as the rank
|
||||
pushinfo.pushdate, # Use pushdate as the rank
|
||||
client_id,
|
||||
access_token,
|
||||
self.log_obj,
|
||||
@ -1431,7 +1400,7 @@ or run without that action (ie: --no-{action})"
|
||||
# TODO: Bug 1165980 - these should be in tree
|
||||
routes.extend([
|
||||
"%s.buildbot.branches.%s.%s" % (index, self.branch, self.stage_platform),
|
||||
"%s.buildbot.revisions.%s.%s.%s" % (index, self.query_revision(), self.branch, self.stage_platform),
|
||||
"%s.buildbot.revisions.%s.%s.%s" % (index, revision, self.branch, self.stage_platform),
|
||||
])
|
||||
task = tc.create_task(routes)
|
||||
tc.claim_task(task)
|
||||
|
||||
@ -989,6 +989,16 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
|
||||
self.set_buildbot_property('funsize_info', json.dumps(funsize_info),
|
||||
write_to_file=True)
|
||||
|
||||
def query_repo(self):
|
||||
# Find the name of our repository
|
||||
mozilla_dir = self.config['mozilla_dir']
|
||||
repo = None
|
||||
for repository in self.config['repos']:
|
||||
if repository.get('dest') == mozilla_dir:
|
||||
repo = repository['repo']
|
||||
break
|
||||
return repo
|
||||
|
||||
def taskcluster_upload(self):
|
||||
auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file'])
|
||||
credentials = {}
|
||||
@ -1016,6 +1026,10 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
|
||||
branch = self.config['branch']
|
||||
platform = self.config['platform']
|
||||
revision = self._query_revision()
|
||||
repo = self.query_repo()
|
||||
if not repo:
|
||||
self.fatal("Unable to determine repository for querying the push info.")
|
||||
pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool')
|
||||
|
||||
routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
|
||||
'testing/taskcluster/routes.json')
|
||||
@ -1038,10 +1052,10 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
|
||||
}
|
||||
fmt.update(self.buildid_to_dict(self._query_buildid()))
|
||||
routes.append(template.format(**fmt))
|
||||
self.info('Using routes: %s' % routes)
|
||||
|
||||
self.info('Using routes: %s' % routes)
|
||||
tc = Taskcluster(branch,
|
||||
self.query_pushdate(),
|
||||
pushinfo.pushdate, # Use pushdate as the rank
|
||||
client_id,
|
||||
access_token,
|
||||
self.log_obj,
|
||||
@ -1057,48 +1071,6 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
|
||||
tc.create_artifact(task, upload_file)
|
||||
tc.report_completed(task)
|
||||
|
||||
def query_pushdate(self):
|
||||
if self.pushdate:
|
||||
return self.pushdate
|
||||
|
||||
mozilla_dir = self.config['mozilla_dir']
|
||||
repo = None
|
||||
for repository in self.config['repos']:
|
||||
if repository.get('dest') == mozilla_dir:
|
||||
repo = repository['repo']
|
||||
break
|
||||
|
||||
if not repo:
|
||||
self.fatal("Unable to determine repository for querying the pushdate.")
|
||||
try:
|
||||
url = '%s/json-pushes?changeset=%s' % (
|
||||
repo,
|
||||
self._query_revision(),
|
||||
)
|
||||
self.info('Pushdate URL is: %s' % url)
|
||||
contents = self.retry(self.load_json_from_url, args=(url,))
|
||||
|
||||
# The contents should be something like:
|
||||
# {
|
||||
# "28537": {
|
||||
# "changesets": [
|
||||
# "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5",
|
||||
# ],
|
||||
# "date": 1428072488,
|
||||
# "user": "user@mozilla.com"
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# So we grab the first element ("28537" in this case) and then pull
|
||||
# out the 'date' field.
|
||||
self.pushdate = contents.itervalues().next()['date']
|
||||
self.info('Pushdate is: %s' % self.pushdate)
|
||||
except Exception:
|
||||
self.exception("Failed to get pushdate from hg.mozilla.org")
|
||||
raise
|
||||
|
||||
return self.pushdate
|
||||
|
||||
# main {{{
|
||||
if __name__ == '__main__':
|
||||
single_locale = DesktopSingleLocale()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user