Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-25 13:51:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1284005 - Replace timestamps with relative timestamps. r=dustin

Browse Source 
--HG--
extra : rebase_source : 0628ff7f93cab9e690fd9d7015783cbc6048d058
This commit is contained in:
Kalpesh Krishna 2016-07-06 03:03:00 -04:00
parent 6a91e5346c
commit 110a36bb4b
29 changed files with 149 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
taskcluster/ci/docker-image/image.yml
View File

@ -1,7 +1,9 @@
---
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: "0 seconds"
deadline:
relative-datestamp: "24 hours"
metadata:
name: 'Docker Image Build: {{image_name}}'
description: 'Build the docker image {{image_name}} for use by dependent tasks'
@ -43,7 +45,8 @@ task:
'{{artifact_path}}':
type: 'file'
path: '/artifacts/image.tar'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: "1 year"
extra:
treeherderEnv:
- staging

6
taskcluster/ci/legacy/tasks/build.yml
View File

@ -4,8 +4,10 @@
taskId: {{build_slugid}}
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: "0 seconds"
deadline:
relative-datestamp: "24 hours"
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com

3
taskcluster/ci/legacy/tasks/builds/android_checkstyle.yml
View File

@ -50,7 +50,8 @@ task:
'public/android/checkstyle/checkstyle.xml':
type: file
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/reports/checkstyle/checkstyle.xml'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: "1 year"
extra:
treeherderEnv:

9
taskcluster/ci/legacy/tasks/builds/android_lint.yml
View File

@ -50,15 +50,18 @@ task:
'public/android/lint/lint-results-automationDebug.html':
type: file
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/outputs/lint-results-automationDebug.html'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: "1 year"
'public/android/lint/lint-results-automationDebug.xml':
type: file
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/outputs/lint-results-automationDebug.xml'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: "1 year"
'public/android/lint/lint-results-automationDebug_files':
type: directory
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/outputs/lint-results-automationDebug_files'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: "1 year"
extra:
treeherderEnv:

3
taskcluster/ci/legacy/tasks/builds/android_test.yml
View File

@ -50,7 +50,8 @@ task:
'public/android/unittest':
type: directory
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/reports/tests'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: "1 year"
extra:
treeherderEnv:

3
taskcluster/ci/legacy/tasks/builds/linux64_clang.yml
View File

@ -34,7 +34,8 @@ task:
'public/clang.tar.xz':
type: file
path: '/home/worker/workspace/artifacts/clang.tar.xz'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: "1 year"
extra:
treeherderEnv:

3
taskcluster/ci/legacy/tasks/builds/linux64_gcc.yml
View File

@ -27,7 +27,8 @@ task:
'public/gcc.tar.xz':
type: file
path: '/home/worker/workspace/artifacts/gcc.tar.xz'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: "1 year"
extra:
treeherderEnv:

3
taskcluster/ci/legacy/tasks/docker_build.yml
View File

@ -23,4 +23,5 @@ task:
'public/build':
type: directory
path: '/home/worker/artifacts/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'

6
taskcluster/ci/legacy/tasks/docker_test.yml
View File

@ -20,8 +20,10 @@ task:
'public/logs/':
type: directory
path: '/home/worker/workspace/build/upload/logs/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
'public/test_info/':
type: directory
path: '/home/worker/workspace/build/blobber_upload_dir/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'

9
taskcluster/ci/legacy/tasks/harness_test.yml
View File

@ -6,8 +6,10 @@
taskId: {{build_slugid}}
docker-image: desktop-build
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com
@ -40,7 +42,8 @@ task:
'public/logs/':
type: directory
path: '/home/worker/workspace/build/upload/logs/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
command:
- bash
- /home/worker/bin/build.sh

9
taskcluster/ci/legacy/tasks/image.yml
View File

@ -4,8 +4,10 @@
taskId: '{{image_slugid}}'
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
metadata:
name: 'Docker Artifact Image Builder'
description: 'Builder for docker images as artifacts'
@ -40,7 +42,8 @@ task:
'{{artifact_path}}':
type: 'file'
path: '/artifacts/image.tar'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
extra:
treeherderEnv:
- staging

9
taskcluster/ci/legacy/tasks/l10n.yml
View File

@ -4,8 +4,10 @@
taskId: {{build_slugid}}
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com
@ -34,7 +36,8 @@ task:
'public/build':
type: directory
path: '/home/worker/workspace/build/upload/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
env:

6
taskcluster/ci/legacy/tasks/lint.yml
View File

@ -4,8 +4,10 @@
taskId: {{build_slugid}}
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
metadata:
source: http://todo.com/soon
owner: mozilla-taskcluster-maintenance@mozilla.com

12
taskcluster/ci/legacy/tasks/phone_build.yml
View File

@ -4,8 +4,10 @@
taskId: {{build_slugid}}
docker-image: phone-builder
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com
@ -44,11 +46,13 @@ task:
'private/build':
type: directory
path: '/home/worker/artifacts/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
'public/build':
type: directory
path: '/home/worker/artifacts-public/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
env:

9
taskcluster/ci/legacy/tasks/post-builds/mulet_simulator.yml
View File

@ -3,8 +3,10 @@
taskId: {{build_slugid}}
docker-image: builder
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
metadata:
source: '{{source}}'
owner: apoirot@mozilla.com
@ -45,7 +47,8 @@ task:
'public/build':
type: directory
path: '/home/worker/artifacts/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
env:
GECKO_BASE_REPOSITORY: '{{base_repository}}'

6
taskcluster/ci/legacy/tasks/post-builds/upload_symbols.yml
View File

@ -3,8 +3,10 @@
taskId: {{taskId}}
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
metadata:
source: '{{source}}'
owner: amiyaguchi@mozilla.com

6
taskcluster/ci/legacy/tasks/test.yml
View File

@ -1,7 +1,9 @@
# This task is the base for most tests in gecko.
task:
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com

3
taskcluster/ci/legacy/tasks/tests/b2g_unittest_base.yml
View File

@ -28,4 +28,5 @@ task:
'public/build':
type: directory
path: '/home/worker/artifacts/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'

3
taskcluster/ci/legacy/tasks/tests/fx_docker_test_base.yml
View File

@ -29,7 +29,8 @@ task:
'public/test':
type: directory
path: '/home/worker/artifacts/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
features:
allowPtrace: true
extra:

3
taskcluster/ci/legacy/tasks/tests/fx_windows_test_base.yml
View File

@ -18,7 +18,8 @@ task:
-
type: 'directory'
path: 'public\test'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
extra:
treeherder:

3
taskcluster/ci/legacy/tasks/tests/harness_marionette.yml
View File

@ -24,7 +24,8 @@ task:
'public/logs/':
type: directory
path: '/home/worker/workspace/mozharness_workspace/upload/logs/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
extra:
treeherder:
symbol: Mn-h

3
taskcluster/ci/legacy/tasks/windows_build.yml
View File

@ -8,7 +8,8 @@ task:
-
type: 'directory'
path: 'public\build'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
extra:
treeherderEnv:
- production

6
taskcluster/ci/legacy/tasks/windows_test.yml
View File

@ -7,8 +7,10 @@ task:
-
type: 'directory'
path: 'public\logs'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
-
type: 'directory'
path: 'public\test_info'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'

6
taskcluster/docs/taskgraph.rst
View File

@ -188,6 +188,12 @@ Each task has the following properties:
``task``
The task's TaskCluster task definition.
The task definition may contain "relative datestamps" of the form
``{"relative-datestamp": "certain number of seconds/hours/days/years"}``.
These will be replaced in the last step, while creating tasks.
The UTC timestamp at that moment is noted, and all the relative datestamps
are replaced with respect to this timestamp.
The task definition may contain "task references" of the form
``{"task-reference": "string containing <task-label>"}``. These will be
replaced during the optimization step, with the appropriate taskId substituted

24
taskcluster/taskgraph/create.py
View File

@ -12,6 +12,10 @@ import os
import logging
from slugid import nice as slugid
from taskgraph.util.time import (
current_json_time,
json_time_from_now
)
logger = logging.getLogger(__name__)
@ -39,7 +43,6 @@ def create_tasks(taskgraph, label_to_taskid):
# that.
for task_id in taskgraph.graph.visit_postorder():
task_def = taskgraph.tasks[task_id].task
# if this task has no dependencies, make it depend on this decision
# task so that it does not start immediately; and so that if this loop
# fails halfway through, none of the already-created tasks run.
@ -65,6 +68,11 @@ def create_tasks(taskgraph, label_to_taskid):
def _create_task(session, task_id, label, task_def):
# create the task using 'http://taskcluster/queue', which is proxied to the queue service
# with credentials appropriate to this job.
# Resolve timestamps
now = current_json_time(datetime_format=True)
task_def = resolve_timestamps(now, task_def)
logger.debug("Creating task with taskId {} for {}".format(task_id, label))
res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id),
data=json.dumps(task_def))
@ -74,3 +82,17 @@ def _create_task(session, task_id, label, task_def):
except:
logger.error(res.text)
res.raise_for_status()
def resolve_timestamps(now, task_def):
def recurse(val):
if isinstance(val, list):
return [recurse(v) for v in val]
elif isinstance(val, dict):
if val.keys() == ['relative-datestamp']:
return json_time_from_now(val['relative-datestamp'], now)
else:
return {k: recurse(v) for k, v in val.iteritems()}
else:
return val
return recurse(task_def)

6
taskcluster/taskgraph/kind/docker_image.py
View File

@ -17,10 +17,6 @@ from taskgraph.util.docker import (
generate_context_hash
)
from taskgraph.util.templates import Templates
from taskgraph.util.time import (
json_time_from_now,
current_json_time,
)
logger = logging.getLogger(__name__)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
@ -54,8 +50,6 @@ class DockerImageTask(base.Task):
'head_rev': params['head_rev'],
'owner': params['owner'],
'level': params['level'],
'from_now': json_time_from_now,
'now': current_json_time(),
'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
.format(repo=params['head_repository'], rev=params['head_rev']),
}

26
taskcluster/taskgraph/kind/legacy.py
View File

@ -197,11 +197,19 @@ def query_vcs_info(repository, revision):
return None
def set_expiration(task, timestamp):
def set_expiration(task, relative_datestamp):
task_def = task['task']
task_def['expires'] = timestamp
if task_def.get('deadline', timestamp) > timestamp:
task_def['deadline'] = timestamp
task_def['expires'] = {'relative-datestamp': relative_datestamp}
if 'deadline' in task_def:
now = current_json_time(datetime_format=True)
timestamp = json_time_from_now(input_str=TRY_EXPIRATION,
now=now,
datetime_format=True)
deadline = json_time_from_now(input_str=task_def['deadline']['relative-datestamp'],
now=now,
datetime_format=True)
if deadline > timestamp:
task_def['deadline']['relative-datestamp'] = relative_datestamp
try:
artifacts = task_def['payload']['artifacts']
@ -212,7 +220,7 @@ def set_expiration(task, timestamp):
# for generic-worker, artifacts is a list
# for taskcluster-worker, it will depend on what we do in artifacts plugin
for artifact in artifacts.values() if hasattr(artifacts, "values") else artifacts:
artifact['expires'] = timestamp
artifact['expires']['relative-datestamp'] = relative_datestamp
def format_treeherder_route(destination, project, revision, pushlog_id):
@ -362,8 +370,6 @@ class LegacyTask(base.Task):
'rank': push_epoch,
'owner': params['owner'],
'level': params['level'],
'from_now': json_time_from_now,
'now': current_json_time(),
}.items())
routes_file = os.path.join(root, 'routes.json')
@ -461,7 +467,7 @@ class LegacyTask(base.Task):
# try builds don't use cache
if project == "try":
remove_caches_from_task(build_task)
set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
set_expiration(build_task, TRY_EXPIRATION)
decorate_task_treeherder_routes(build_task['task'],
build_parameters['project'],
@ -544,7 +550,7 @@ class LegacyTask(base.Task):
set_interactive_task(post_task, interactive)
if project == "try":
set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
set_expiration(post_task, TRY_EXPIRATION)
post_task['attributes'] = attributes.copy()
post_task['attributes']['legacy_kind'] = 'post_build'
@ -595,7 +601,7 @@ class LegacyTask(base.Task):
test_parameters['pushlog_id'])
if project == "try":
set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))
set_expiration(test_task, TRY_EXPIRATION)
test_task['attributes'] = attributes.copy()
test_task['attributes']['legacy_kind'] = 'unittest'

21
taskcluster/taskgraph/util/time.py
View File

@ -82,10 +82,11 @@ def value_of(input_str):
return ALIASES[unit](value)
def json_time_from_now(input_str, now=None):
def json_time_from_now(input_str, now=None, datetime_format=False):
'''
:param str input_str: Input string (see value of)
:param datetime now: Optionally set the definition of `now`
:param boolean datetime_format: Set `True` to get a `datetime` output
:returns: JSON string representation of time in future.
'''
@ -94,14 +95,20 @@ def json_time_from_now(input_str, now=None):
time = now + value_of(input_str)
# Sorta a big hack but the json schema validator for date does not like the
# ISO dates until 'Z' (for timezone) is added...
return time.isoformat() + 'Z'
if datetime_format is True:
return time
else:
# Sorta a big hack but the json schema validator for date does not like the
# ISO dates until 'Z' (for timezone) is added...
return time.isoformat() + 'Z'
def current_json_time():
def current_json_time(datetime_format=False):
'''
:param boolean datetime_format: Set `True` to get a `datetime` output
:returns: JSON string representation of the current time.
'''
return datetime.datetime.utcnow().isoformat() + 'Z'
if datetime_format is True:
return datetime.datetime.utcnow()
else:
return datetime.datetime.utcnow().isoformat() + 'Z'

3
testing/docker/phone-builder/tests/valid.yml
View File

@ -35,7 +35,8 @@ task:
'private/build':
type: directory
path: '/home/worker/artifacts/'
expires: '{{#from_now}}1 year{{/from_now}}'
expires:
relative-datestamp: '1 year'
extra:
# Rather then enforcing particular conventions we require that all build