Bug 1616925 - Support a taskcluster-based ssh key for fetch jobs r=tomprince

Differential Revision: https://phabricator.services.mozilla.com/D81448

taskcluster/docker/fetch/Dockerfile

@@ -18,6 +18,7 @@ RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAG
       gnupg \
       bzip2 \
       git \
+      openssh-client \
       python3-requests \
       python3-zstandard \
       unzip

taskcluster/scripts/misc/fetch-content

@@ -483,7 +483,7 @@ def fetch_urls(downloads):
 
 
 def git_checkout_archive(dest_path: pathlib.Path, repo: str, commit: str,
-                         prefix=None):
+                         prefix=None, ssh_key=None):
     """Produce an archive of the files comprising a Git checkout."""
     dest_path.parent.mkdir(parents=True, exist_ok=True)
 
@@ -501,8 +501,23 @@ def git_checkout_archive(dest_path: pathlib.Path, repo: str, commit: str,
         # to initiate a clone. Since the commit-ish may not refer to a ref, we
         # simply perform a full clone followed by a checkout.
         print('cloning %s to %s' % (repo, git_dir))
+
+        env = os.environ.copy()
+        keypath = ""
+        if ssh_key:
+            taskcluster_secret_url = api(os.environ.get('TASKCLUSTER_PROXY_URL'), "secrets", "v1", "secret/{keypath}".format(keypath=ssh_key))
+            taskcluster_secret = b''.join(stream_download(taskcluster_secret_url))
+            taskcluster_secret = json.loads(taskcluster_secret)
+            sshkey = taskcluster_secret['secret']['ssh_privkey']
+
+            keypath = temp_dir.joinpath("ssh-key")
+            keypath.write_text(sshkey)
+            keypath.chmod(0o600)
+
+            env = {"GIT_SSH_COMMAND": "ssh -o 'StrictHostKeyChecking no' -i {keypath}".format(keypath=keypath)}
+
         subprocess.run(['git', 'clone', '-n', repo, str(git_dir)],
-                       check=True)
+                       check=True, env=env)
 
         subprocess.run(['git', 'checkout', commit],
                        cwd=str(git_dir), check=True)
@@ -510,6 +525,9 @@ def git_checkout_archive(dest_path: pathlib.Path, repo: str, commit: str,
         subprocess.run(['git', 'submodule', 'update', '--init'],
                        cwd=str(git_dir), check=True)
 
+        if keypath:
+            os.remove(keypath)
+
         print('creating archive %s of commit %s' % (dest_path, commit))
         proc = subprocess.Popen([
             'tar', 'cf', '-', '--exclude=.git', '-C', str(temp_dir), prefix,
@@ -527,7 +545,7 @@ def command_git_checkout_archive(args):
 
     try:
         git_checkout_archive(dest, args.repo, args.commit,
-                             prefix=args.path_prefix)
+                             prefix=args.path_prefix, ssh_key=args.ssh_key_secret)
     except Exception:
         try:
             dest.unlink()
@@ -640,6 +658,8 @@ def main():
                               help='Git commit to check out')
     git_checkout.add_argument('dest',
                               help='Destination path of archive')
+    git_checkout.add_argument('--ssh-key-secret',
+                              help='The scope path of the ssh key to used for checkout')
 
     url = subparsers.add_parser('static-url', help='Download a static URL')
     url.set_defaults(func=command_static_url)

taskcluster/taskgraph/transforms/fetch.py

@@ -183,6 +183,10 @@ def make_task(config, jobs):
             },
         }
 
+        if job.get('secret', None):
+            task['scopes'] = ["secrets:get:" + job.get('secret')]
+            task['worker']['taskcluster-proxy'] = True
+
         if not taskgraph.fast:
             cache_name = task['label'].replace('{}-'.format(config.kind), '', 1)
 
@@ -295,6 +299,11 @@ def create_fetch_url_task(config, name, fetch):
     Required('revision'): text_type,
     Optional('artifact-name'): text_type,
     Optional('path-prefix'): text_type,
+    # ssh-key is a taskcluster secret path (e.g. project/civet/github-deploy-key)
+    # In the secret dictionary, the key should be specified as
+    #  "ssh_privkey": "-----BEGIN OPENSSH PRIVATE KEY-----\nkfksnb3jc..."
+    # n.b. The OpenSSH private key file format requires a newline at the end of the file.
+    Optional('ssh-key'): text_type,
 })
 def create_git_fetch_task(config, name, fetch):
     path_prefix = fetch.get('path-prefix')
@@ -318,10 +327,16 @@ def create_git_fetch_task(config, name, fetch):
         '/builds/worker/artifacts/%s' % artifact_name,
     ]
 
+    ssh_key = fetch.get('ssh-key')
+    if ssh_key:
+        args.append('--ssh-key-secret')
+        args.append(ssh_key)
+
     return {
         'command': args,
         'artifact_name': artifact_name,
         'digest_data': [fetch['revision'], path_prefix, artifact_name],
+        'secret': ssh_key
     }