Bug 1436037 - [taskgraph] Support Windows generic-worker with run-task, r=gps

This enables Windows generic-worker based tasks to use the run-task script.

Differential Revision: https://phabricator.services.mozilla.com/D10758

--HG--
extra : moz-landing-system : lando

taskcluster/scripts/run-task

@@ -448,13 +448,14 @@ def vcs_checkout(source_repo, dest, store_path,
     if sparse_profile:
         args.extend(['--sparseprofile', sparse_profile])
 
+    dest = os.path.abspath(dest)
     args.extend([
         revision_flag, revision_value,
         source_repo, dest,
     ])
 
     res = run_and_prefix_output(b'vcs', args,
-                                extra_env={b'PYTHONUNBUFFERED': b'1'})
+                                extra_env={'PYTHONUNBUFFERED': '1'})
     if res:
         sys.exit(res)
 
@@ -741,6 +742,9 @@ def main(args):
         return 1
 
     try:
+        if 'GECKO_PATH' in os.environ:
+            os.environ['GECKO_PATH'] = os.path.abspath(os.environ['GECKO_PATH'])
+
         if 'MOZ_FETCHES' in os.environ:
             fetch_artifacts()
 

taskcluster/taskgraph/transforms/job/common.py

@@ -64,10 +64,20 @@ def support_vcs_checkout(config, job, taskdesc, sparse=False):
     This can only be used with ``run-task`` tasks, as the cache name is
     reserved for ``run-task`` tasks.
     """
-    level = config.params['level']
+    is_win = job['worker']['os'] == 'windows'
 
-    # native-engine does not support caches (yet), so we just do a full clone
-    # every time :(
+    if is_win:
+        checkoutdir = './build'
+        geckodir = '{}/src'.format(checkoutdir)
+        hgstore = 'y:/hg-shared'
+    else:
+        checkoutdir = '{workdir}/checkouts'.format(**job['run'])
+        geckodir = '{}/gecko'.format(checkoutdir)
+        hgstore = '{}/hg-store'.format(checkoutdir)
+
+    level = config.params['level']
+    # native-engine and generic-worker do not support caches (yet), so we just
+    # do a full clone every time :(
     if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
         name = 'level-%s-checkouts' % level
 
@@ -84,15 +94,15 @@ def support_vcs_checkout(config, job, taskdesc, sparse=False):
         taskdesc['worker'].setdefault('caches', []).append({
             'type': 'persistent',
             'name': name,
-            'mount-point': '{workdir}/checkouts'.format(**job['run']),
+            'mount-point': checkoutdir,
         })
 
     taskdesc['worker'].setdefault('env', {}).update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
-        'GECKO_PATH': '{workdir}/checkouts/gecko'.format(**job['run']),
-        'HG_STORE_PATH': '{workdir}/checkouts/hg-store'.format(**job['run']),
+        'GECKO_PATH': geckodir,
+        'HG_STORE_PATH': hgstore,
     })
 
     if 'comm_base_repository' in config.params:

taskcluster/taskgraph/transforms/job/mach.py

@@ -30,14 +30,19 @@ mach_schema = Schema({
 })
 
 
-@run_job_using("docker-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False})
-@run_job_using("native-engine", "mach", schema=mach_schema, defaults={'comm-checkout': False})
-@run_job_using("generic-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False})
-def docker_worker_mach(config, job, taskdesc):
+defaults = {
+    'comm-checkout': False,
+}
+
+
+@run_job_using("docker-worker", "mach", schema=mach_schema, defaults=defaults)
+@run_job_using("native-engine", "mach", schema=mach_schema, defaults=defaults)
+@run_job_using("generic-worker", "mach", schema=mach_schema, defaults=defaults)
+def configure_mach(config, job, taskdesc):
     run = job['run']
 
     # defer to the run_task implementation
-    run['command'] = 'cd {workdir}/checkouts/gecko && ./mach {mach}'.format(**run)
+    run['command'] = 'cd $GECKO_PATH && ./mach {mach}'.format(**run)
     run['using'] = 'run-task'
     del run['mach']
     configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])

taskcluster/taskgraph/transforms/job/python_test.py

@@ -25,12 +25,15 @@ python_test_schema = Schema({
 })
 
 
-@run_job_using(
-    'docker-worker',
-    'python-test',
-    schema=python_test_schema,
-    defaults={'python-version': 2, 'subsuite': 'default'})
-def docker_worker_python_test(config, job, taskdesc):
+defaults = {
+    'python-version': 2,
+    'subsuite': 'default',
+}
+
+
+@run_job_using('docker-worker', 'python-test', schema=python_test_schema, defaults=defaults)
+@run_job_using('generic-worker', 'python-test', schema=python_test_schema, defaults=defaults)
+def configure_python_test(config, job, taskdesc):
     run = job['run']
 
     # defer to the mach implementation

taskcluster/taskgraph/transforms/job/run_task.py

@@ -41,12 +41,12 @@ run_task_schema = Schema({
 })
 
 
-def common_setup(config, job, taskdesc, command, checkoutdir):
+def common_setup(config, job, taskdesc, command, geckodir):
     run = job['run']
     if run['checkout']:
         support_vcs_checkout(config, job, taskdesc,
                              sparse=bool(run['sparse-profile']))
-        command.append('--vcs-checkout={}/gecko'.format(checkoutdir))
+        command.append('--vcs-checkout={}'.format(geckodir))
 
     if run['sparse-profile']:
         command.append('--sparse-profile=build/sparse-profiles/%s' %
@@ -73,7 +73,8 @@ def docker_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     command = ['/builds/worker/bin/run-task']
-    common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run))
+    common_setup(config, job, taskdesc, command,
+                 geckodir='{workdir}/checkouts/gecko'.format(**run))
 
     if run.get('cache-dotcache'):
         worker['caches'].append({
@@ -100,7 +101,8 @@ def native_engine_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     command = ['./run-task']
-    common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run))
+    common_setup(config, job, taskdesc, command,
+                 geckodir='{workdir}/checkouts/gecko'.format(**run))
 
     worker['context'] = run_task_url(config)
 
@@ -119,8 +121,16 @@ def native_engine_run_task(config, job, taskdesc):
 def generic_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
-    command = ['./run-task']
-    common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run))
+    is_win = worker['os'] == 'windows'
+
+    if is_win:
+        command = ['C:/mozilla-build/python3/python3.exe', 'run-task']
+        geckodir = './build/src'
+    else:
+        command = ['./run-task']
+        geckodir = '{workdir}/checkouts/gecko'.format(**run)
+
+    common_setup(config, job, taskdesc, command, geckodir=geckodir)
 
     worker.setdefault('mounts', [])
     if run.get('cache-dotcache'):
@@ -137,7 +147,17 @@ def generic_worker_run_task(config, job, taskdesc):
 
     run_command = run['command']
     if isinstance(run_command, basestring):
+        if is_win:
+            run_command = '"{}"'.format(run_command)
         run_command = ['bash', '-cx', run_command]
+
     command.append('--')
     command.extend(run_command)
-    worker['command'] = [['chmod', '+x', 'run-task'], command]
+
+    if is_win:
+        worker['command'] = [' '.join(command)]
+    else:
+        worker['command'] = [
+            ['chmod', '+x', 'run-task'],
+            command,
+        ]

taskcluster/taskgraph/transforms/task.py

@@ -1880,8 +1880,8 @@ def check_caches_are_volumes(task):
         return
 
     raise Exception('task %s (image %s) has caches that are not declared as '
-                    'Docker volumes: %s'
-                    'Have you added them as VOLUMEs in the Dockerfile?'
+                    'Docker volumes: %s '
+                    '(have you added them as VOLUMEs in the Dockerfile?)'
                     % (task['label'], task['worker']['docker-image'],
                        ', '.join(sorted(missing))))