2016-07-30 03:53:21 +00:00
|
|
|
#!/usr/bin/python2.7 -u
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
|
|
|
|
"""Run a task after performing common actions.
|
|
|
|
|
|
|
|
This script is meant to be the "driver" for TaskCluster based tasks.
|
|
|
|
It receives some common arguments to control the run-time environment.
|
|
|
|
|
|
|
|
It performs actions as requested from the arguments. Then it executes
|
|
|
|
the requested process and prints its output, prefixing it with the
|
|
|
|
current time to improve log usefulness.
|
|
|
|
"""
|
|
|
|
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
import datetime
|
|
|
|
import errno
|
|
|
|
import grp
|
2016-09-01 22:38:30 +00:00
|
|
|
import json
|
2016-07-30 03:53:21 +00:00
|
|
|
import os
|
|
|
|
import pwd
|
2016-08-09 20:14:05 +00:00
|
|
|
import re
|
2016-09-14 19:26:15 +00:00
|
|
|
import stat
|
2016-07-30 03:53:21 +00:00
|
|
|
import subprocess
|
|
|
|
import sys
|
2016-09-01 22:38:30 +00:00
|
|
|
import urllib2
|
|
|
|
|
|
|
|
|
|
|
|
FINGERPRINT_URL = 'http://taskcluster/secrets/v1/secret/project/taskcluster/gecko/hgfingerprint'
|
2016-10-17 14:45:02 +00:00
|
|
|
FALLBACK_FINGERPRINT = {
|
|
|
|
'fingerprints':
|
|
|
|
"sha256:8e:ad:f7:6a:eb:44:06:15:ed:f3:e4:69:a6:64:60:37:2d:ff:98:88:37"
|
|
|
|
":bf:d7:b8:40:84:01:48:9c:26:ce:d9"}
|
2016-07-30 03:53:21 +00:00
|
|
|
|
|
|
|
|
|
|
|
def print_line(prefix, m):
|
|
|
|
now = datetime.datetime.utcnow()
|
2016-08-10 23:01:25 +00:00
|
|
|
print(b'[%s %sZ] %s' % (prefix, now.isoformat(), m), end=b'')
|
2016-07-30 03:53:21 +00:00
|
|
|
|
|
|
|
|
2016-09-23 06:33:01 +00:00
|
|
|
def run_and_prefix_output(prefix, args, extra_env=None):
|
2016-07-30 03:53:21 +00:00
|
|
|
"""Runs a process and prefixes its output with the time.
|
|
|
|
|
|
|
|
Returns the process exit code.
|
|
|
|
"""
|
2016-08-10 23:01:25 +00:00
|
|
|
print_line(prefix, b'executing %s\n' % args)
|
2016-07-30 03:53:21 +00:00
|
|
|
|
2016-09-23 06:33:01 +00:00
|
|
|
env = dict(os.environ)
|
|
|
|
env.update(extra_env or {})
|
|
|
|
|
2016-07-30 03:53:21 +00:00
|
|
|
# Note: TaskCluster's stdin is a TTY. This attribute is lost
|
|
|
|
# when we pass sys.stdin to the invoked process. If we cared
|
|
|
|
# to preserve stdin as a TTY, we could make this work. But until
|
|
|
|
# someone needs it, don't bother.
|
|
|
|
p = subprocess.Popen(args,
|
|
|
|
# Disable buffering because we want to receive output
|
|
|
|
# as it is generated so timestamps in logs are
|
|
|
|
# accurate.
|
|
|
|
bufsize=0,
|
|
|
|
stdout=subprocess.PIPE,
|
|
|
|
stderr=subprocess.STDOUT,
|
|
|
|
stdin=sys.stdin.fileno(),
|
|
|
|
cwd='/',
|
2016-09-23 06:33:01 +00:00
|
|
|
env=env,
|
2016-07-30 03:53:21 +00:00
|
|
|
# So \r in progress bars are rendered as multiple
|
|
|
|
# lines, preserving progress indicators.
|
|
|
|
universal_newlines=True)
|
|
|
|
|
|
|
|
while True:
|
|
|
|
data = p.stdout.readline()
|
2016-08-10 23:01:25 +00:00
|
|
|
if data == b'':
|
2016-07-30 03:53:21 +00:00
|
|
|
break
|
|
|
|
|
|
|
|
print_line(prefix, data)
|
|
|
|
|
2016-08-02 04:44:22 +00:00
|
|
|
return p.wait()
|
2016-07-30 03:53:21 +00:00
|
|
|
|
|
|
|
|
2016-09-30 00:05:05 +00:00
|
|
|
def vcs_checkout(source_repo, dest, base_repo=None, revision=None, branch=None):
|
2016-08-09 20:14:05 +00:00
|
|
|
# Specify method to checkout a revision. This defaults to revisions as
|
|
|
|
# SHA-1 strings, but also supports symbolic revisions like `tip` via the
|
|
|
|
# branch flag.
|
2016-09-30 00:05:05 +00:00
|
|
|
if revision:
|
2016-08-09 20:14:05 +00:00
|
|
|
revision_flag = b'--revision'
|
2016-09-30 00:05:05 +00:00
|
|
|
revision_value = revision
|
|
|
|
elif branch:
|
2016-08-09 20:14:05 +00:00
|
|
|
revision_flag = b'--branch'
|
2016-09-30 00:05:05 +00:00
|
|
|
revision_value = branch
|
2016-08-09 20:14:05 +00:00
|
|
|
else:
|
|
|
|
print('revision is not specified for checkout')
|
|
|
|
sys.exit(1)
|
|
|
|
|
2016-09-01 22:38:30 +00:00
|
|
|
# Obtain certificate fingerprints.
|
|
|
|
try:
|
|
|
|
print_line(b'vcs', 'fetching hg.mozilla.org fingerprint from %s\n' %
|
|
|
|
FINGERPRINT_URL)
|
|
|
|
res = urllib2.urlopen(FINGERPRINT_URL, timeout=10)
|
|
|
|
secret = res.read()
|
2016-10-17 14:45:02 +00:00
|
|
|
try:
|
|
|
|
secret = json.loads(secret, encoding='utf-8')
|
|
|
|
except ValueError:
|
|
|
|
print_line(b'vcs', 'invalid JSON in hg fingerprint secret')
|
|
|
|
sys.exit(1)
|
|
|
|
except urllib2.URLError:
|
|
|
|
print_line(b'vcs', 'Unable to retrieve current hg.mozilla.org fingerprint'
|
|
|
|
'using the secret service, using fallback instead.')
|
|
|
|
# XXX This fingerprint will not be accurate if running on an old
|
|
|
|
# revision after the server fingerprint has changed.
|
|
|
|
secret = {'secret': FALLBACK_FINGERPRINT}
|
2016-09-01 22:38:30 +00:00
|
|
|
|
|
|
|
hgmo_fingerprint = secret['secret']['fingerprints'].encode('ascii')
|
|
|
|
|
2016-10-18 16:46:55 +00:00
|
|
|
# Put the shared stores as a sibling of the checkout directory. This ensures
|
|
|
|
# they are on the same cache.
|
|
|
|
share_base = os.path.normpath(os.path.join(os.path.dirname(dest), b'hg-shared'))
|
|
|
|
|
2016-09-30 00:05:05 +00:00
|
|
|
args = [
|
2016-09-01 22:38:30 +00:00
|
|
|
b'/usr/bin/hg',
|
|
|
|
b'--config', b'hostsecurity.hg.mozilla.org:fingerprints=%s' % hgmo_fingerprint,
|
|
|
|
b'robustcheckout',
|
2016-10-18 16:46:55 +00:00
|
|
|
b'--sharebase', share_base,
|
2016-07-30 03:53:21 +00:00
|
|
|
b'--purge',
|
2016-09-30 00:05:05 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
if base_repo:
|
|
|
|
args.extend([b'--upstream', base_repo])
|
|
|
|
|
|
|
|
args.extend([
|
|
|
|
revision_flag, revision_value,
|
|
|
|
source_repo, dest,
|
|
|
|
])
|
2016-07-30 03:53:21 +00:00
|
|
|
|
2016-09-30 00:05:05 +00:00
|
|
|
res = run_and_prefix_output(b'vcs', args,
|
|
|
|
extra_env={b'PYTHONUNBUFFERED': b'1'})
|
2016-07-30 03:53:21 +00:00
|
|
|
if res:
|
|
|
|
sys.exit(res)
|
|
|
|
|
2016-10-17 20:48:33 +00:00
|
|
|
# Meant to be temporary to flush out what's going on in bug 1292071.
|
|
|
|
print_line(b'vcs', b'verifying working directory is clean\n')
|
|
|
|
status = subprocess.check_output([b'/usr/bin/hg', b'status', b'--all'],
|
|
|
|
cwd=dest, stderr=subprocess.STDOUT)
|
|
|
|
status = status.strip()
|
|
|
|
status_lines = [l for l in status.splitlines() if not l.startswith(b'C ')]
|
|
|
|
if status_lines:
|
|
|
|
print_line(b'vcs', b'Error: checkout is not pristine! '
|
|
|
|
b'Report this in bug 1292071\n')
|
|
|
|
for line in status_lines:
|
|
|
|
print_line(b'vcs', b'%s\n' % line)
|
|
|
|
sys.exit(1)
|
|
|
|
|
2016-08-09 20:14:05 +00:00
|
|
|
# Update the current revision hash and ensure that it is well formed.
|
|
|
|
revision = subprocess.check_output(
|
|
|
|
[b'/usr/bin/hg', b'log',
|
|
|
|
b'--rev', b'.',
|
|
|
|
b'--template', b'{node}'],
|
2016-09-30 00:05:05 +00:00
|
|
|
cwd=dest)
|
2016-08-09 20:14:05 +00:00
|
|
|
|
|
|
|
assert re.match('^[a-f0-9]{40}$', revision)
|
2016-09-30 00:05:05 +00:00
|
|
|
return revision
|
2016-08-09 20:14:05 +00:00
|
|
|
|
2016-07-30 03:53:21 +00:00
|
|
|
|
|
|
|
def main(args):
|
2016-08-10 23:01:25 +00:00
|
|
|
print_line(b'setup', b'run-task started\n')
|
2016-07-30 03:53:21 +00:00
|
|
|
|
|
|
|
if os.getuid() != 0:
|
|
|
|
print('assertion failed: not running as root')
|
|
|
|
return 1
|
|
|
|
|
|
|
|
# Arguments up to '--' are ours. After are for the main task
|
|
|
|
# to be executed.
|
|
|
|
try:
|
|
|
|
i = args.index('--')
|
|
|
|
our_args = args[0:i]
|
|
|
|
task_args = args[i + 1:]
|
|
|
|
except ValueError:
|
|
|
|
our_args = args
|
|
|
|
task_args = []
|
|
|
|
|
|
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument('--user', default='worker', help='user to run as')
|
|
|
|
parser.add_argument('--group', default='worker', help='group to run as')
|
2016-08-17 02:31:12 +00:00
|
|
|
# We allow paths to be chowned by the --user:--group before permissions are
|
|
|
|
# dropped. This is often necessary for caches/volumes, since they default
|
|
|
|
# to root:root ownership.
|
|
|
|
parser.add_argument('--chown', action='append',
|
|
|
|
help='Directory to chown to --user:--group')
|
|
|
|
parser.add_argument('--chown-recursive', action='append',
|
|
|
|
help='Directory to recursively chown to --user:--group')
|
2016-07-30 03:53:21 +00:00
|
|
|
parser.add_argument('--vcs-checkout',
|
|
|
|
help='Directory where Gecko checkout should be created')
|
2016-09-30 01:49:07 +00:00
|
|
|
parser.add_argument('--tools-checkout',
|
|
|
|
help='Directory where build/tools checkout should be created')
|
2016-07-30 03:53:21 +00:00
|
|
|
|
|
|
|
args = parser.parse_args(our_args)
|
|
|
|
|
|
|
|
try:
|
|
|
|
user = pwd.getpwnam(args.user)
|
|
|
|
except KeyError:
|
|
|
|
print('could not find user %s; specify --user to a known user' %
|
|
|
|
args.user)
|
|
|
|
return 1
|
|
|
|
try:
|
|
|
|
group = grp.getgrnam(args.group)
|
|
|
|
except KeyError:
|
|
|
|
print('could not find group %s; specify --group to a known group' %
|
|
|
|
args.group)
|
|
|
|
return 1
|
|
|
|
|
|
|
|
uid = user.pw_uid
|
|
|
|
gid = group.gr_gid
|
|
|
|
|
|
|
|
# Find all groups to which this user is a member.
|
|
|
|
gids = [g.gr_gid for g in grp.getgrall() if args.group in g.gr_mem]
|
|
|
|
|
2016-09-14 19:26:15 +00:00
|
|
|
wanted_dir_mode = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR
|
2016-10-17 14:45:02 +00:00
|
|
|
|
2016-09-14 19:26:15 +00:00
|
|
|
def set_dir_permissions(path, uid, gid):
|
|
|
|
st = os.lstat(path)
|
|
|
|
|
|
|
|
if st.st_uid != uid or st.st_gid != gid:
|
|
|
|
os.chown(path, uid, gid)
|
|
|
|
|
|
|
|
# Also make sure dirs are writable in case we need to delete
|
|
|
|
# them.
|
|
|
|
if st.st_mode & wanted_dir_mode != wanted_dir_mode:
|
|
|
|
os.chmod(path, st.st_mode | wanted_dir_mode)
|
|
|
|
|
2016-08-17 02:31:12 +00:00
|
|
|
# Change ownership of requested paths.
|
|
|
|
# FUTURE: parse argument values for user/group if we don't want to
|
|
|
|
# use --user/--group.
|
|
|
|
for path in args.chown or []:
|
|
|
|
print_line(b'chown', b'changing ownership of %s to %s:%s\n' % (
|
|
|
|
path, user.pw_name, group.gr_name))
|
2016-09-14 19:26:15 +00:00
|
|
|
set_dir_permissions(path, uid, gid)
|
2016-08-17 02:31:12 +00:00
|
|
|
|
|
|
|
for path in args.chown_recursive or []:
|
|
|
|
print_line(b'chown', b'recursively changing ownership of %s to %s:%s\n' %
|
|
|
|
(path, user.pw_name, group.gr_name))
|
2016-09-29 17:07:21 +00:00
|
|
|
|
|
|
|
set_dir_permissions(path, uid, gid)
|
|
|
|
|
2016-08-17 02:31:12 +00:00
|
|
|
for root, dirs, files in os.walk(path):
|
|
|
|
for d in dirs:
|
2016-09-14 19:26:15 +00:00
|
|
|
set_dir_permissions(os.path.join(root, d), uid, gid)
|
|
|
|
|
2016-08-17 02:31:12 +00:00
|
|
|
for f in files:
|
2016-09-30 22:22:28 +00:00
|
|
|
# File may be a symlink that points to nowhere. In which case
|
|
|
|
# os.chown() would fail because it attempts to follow the
|
|
|
|
# symlink. We only care about directory entries, not what
|
|
|
|
# they point to. So setting the owner of the symlink should
|
|
|
|
# be sufficient.
|
|
|
|
os.lchown(os.path.join(root, f), uid, gid)
|
2016-08-17 02:31:12 +00:00
|
|
|
|
2016-09-30 00:05:05 +00:00
|
|
|
def prepare_checkout_dir(checkout):
|
|
|
|
if not checkout:
|
|
|
|
return
|
|
|
|
|
2016-07-30 03:53:21 +00:00
|
|
|
# Ensure the directory for the source checkout exists.
|
|
|
|
try:
|
|
|
|
os.makedirs(os.path.dirname(checkout))
|
|
|
|
except OSError as e:
|
|
|
|
if e.errno != errno.EEXIST:
|
|
|
|
raise
|
|
|
|
|
|
|
|
# And that it is owned by the appropriate user/group.
|
|
|
|
os.chown(os.path.dirname(checkout), uid, gid)
|
|
|
|
|
2016-10-18 16:46:55 +00:00
|
|
|
# And ensure the shared store path (derived from the checkout path)
|
|
|
|
# exists and has proper permissions.
|
|
|
|
share_path = os.path.join(os.path.dirname(checkout), 'hg-shared')
|
|
|
|
try:
|
|
|
|
os.makedirs(share_path)
|
|
|
|
except OSError as e:
|
|
|
|
if e.errno != errno.EEXIST:
|
|
|
|
raise
|
|
|
|
|
|
|
|
os.chown(share_path, uid, gid)
|
|
|
|
|
2016-09-30 00:05:05 +00:00
|
|
|
prepare_checkout_dir(args.vcs_checkout)
|
2016-09-30 01:49:07 +00:00
|
|
|
prepare_checkout_dir(args.tools_checkout)
|
2016-09-30 00:05:05 +00:00
|
|
|
|
2016-07-30 03:53:21 +00:00
|
|
|
# Drop permissions to requested user.
|
|
|
|
# This code is modeled after what `sudo` was observed to do in a Docker
|
|
|
|
# container. We do not bother calling setrlimit() because containers have
|
|
|
|
# their own limits.
|
2016-08-10 23:01:25 +00:00
|
|
|
print_line(b'setup', b'running as %s:%s\n' % (args.user, args.group))
|
2016-07-30 03:53:21 +00:00
|
|
|
os.setgroups(gids)
|
|
|
|
os.umask(022)
|
|
|
|
os.setresgid(gid, gid, gid)
|
|
|
|
os.setresuid(uid, uid, uid)
|
|
|
|
|
2016-08-09 20:14:05 +00:00
|
|
|
# Checkout the repository, setting the GECKO_HEAD_REV to the current
|
|
|
|
# revision hash. Revision hashes have priority over symbolic revisions. We
|
|
|
|
# disallow running tasks with symbolic revisions unless they have been
|
|
|
|
# resolved by a checkout.
|
2016-09-30 00:05:05 +00:00
|
|
|
if args.vcs_checkout:
|
|
|
|
base_repo = os.environ.get('GECKO_BASE_REPOSITORY')
|
|
|
|
# Some callers set the base repository to mozilla-central for historical
|
|
|
|
# reasons. Switch to mozilla-unified because robustcheckout works best
|
|
|
|
# with it.
|
|
|
|
if base_repo == 'https://hg.mozilla.org/mozilla-central':
|
|
|
|
base_repo = b'https://hg.mozilla.org/mozilla-unified'
|
|
|
|
|
|
|
|
os.environ['GECKO_HEAD_REV'] = vcs_checkout(
|
|
|
|
os.environ['GECKO_HEAD_REPOSITORY'],
|
|
|
|
args.vcs_checkout,
|
|
|
|
base_repo=base_repo,
|
|
|
|
revision=os.environ.get('GECKO_HEAD_REV'),
|
|
|
|
branch=os.environ.get('GECKO_HEAD_REF'))
|
|
|
|
|
2016-08-09 20:14:05 +00:00
|
|
|
elif not os.environ.get('GECKO_HEAD_REV') and \
|
|
|
|
os.environ.get('GECKO_HEAD_REF'):
|
|
|
|
print('task should be defined in terms of non-symbolic revision')
|
|
|
|
return 1
|
2016-07-30 03:53:21 +00:00
|
|
|
|
2016-09-30 01:49:07 +00:00
|
|
|
if args.tools_checkout:
|
|
|
|
vcs_checkout(b'https://hg.mozilla.org/build/tools',
|
|
|
|
args.tools_checkout,
|
|
|
|
# Always check out the latest commit on default branch.
|
|
|
|
# This is non-deterministic!
|
|
|
|
branch=b'default')
|
|
|
|
|
2016-08-10 23:01:25 +00:00
|
|
|
return run_and_prefix_output(b'task', task_args)
|
2016-07-30 03:53:21 +00:00
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
# Unbuffer stdio.
|
|
|
|
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
|
|
|
|
sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
|
|
|
|
|
|
|
|
sys.exit(main(sys.argv[1:]))
|