From c9eaaa78324525ebaa360c7bbcdda011f31a3eba Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Fri, 11 Sep 2015 11:35:00 -0400 Subject: [PATCH] Bug 1198179: in taskcluster, have 'make upload' just copy; r=ted.mielczarek --HG-- extra : commitid : C1f3kiibCXn extra : rebase_source : ed715e77014b91fb19c7a6bc7a0a04134f1e69cc extra : amend_source : 46abf6b9bfc0f7993da9814d8f6e3959379c434f --- build/upload.py | 60 +++++++++++++++---- .../configs/builds/build_pool_specifics.py | 6 +- .../mozharness/mozilla/building/buildbase.py | 21 ++++--- .../scripts/builder/build-linux.sh | 33 ++-------- .../tasks/builds/linux64_clobber.yml | 6 +- .../taskcluster/tasks/builds/opt_macosx64.yml | 2 - 6 files changed, 72 insertions(+), 56 deletions(-) diff --git a/build/upload.py b/build/upload.py index 2c70371bae93..475480a402bd 100644 --- a/build/upload.py +++ b/build/upload.py @@ -15,6 +15,10 @@ # If UPLOAD_HOST and UPLOAD_USER are not set, this script will simply write out # the properties file. # +# If UPLOAD_HOST is "localhost", then files are simply copied to UPLOAD_PATH. +# In this case, UPLOAD_TO_TEMP and POST_UPLOAD_CMD are not supported, and no +# properties are written out. +# # And will use the following optional environment variables if set: # UPLOAD_SSH_KEY : path to a ssh private key to use # UPLOAD_PORT : port to use for ssh @@ -32,6 +36,7 @@ import re import json import errno import hashlib +import shutil from optparse import OptionParser from subprocess import check_call, check_output, STDOUT import redo @@ -106,7 +111,7 @@ def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None): raise Exception("Command %s returned non-zero exit code" % cmdline) -def GetRemotePath(path, local_file, base_path): +def GetBaseRelativePath(path, local_file, base_path): """Given a remote path to upload to, a full path to a local file, and an optional full path that is a base path of the local file, construct the full remote path to place the file in. If base_path is not None, include @@ -214,7 +219,7 @@ def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, if not os.path.isfile(file): raise IOError("File not found: %s" % file) # first ensure that path exists remotely - remote_path = GetRemotePath(path, file, base_path) + remote_path = GetBaseRelativePath(path, file, base_path) DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key) if verbose: print "Uploading " + file @@ -236,6 +241,25 @@ def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, print "Upload complete" return properties +def CopyFilesLocally(path, files, verbose=False, base_path=None, package=None): + """Copy each file in the list of files to `path`. The `base_path` argument is treated + as it is by UploadFiles.""" + if not path.endswith("/"): + path += "/" + if base_path is not None: + base_path = os.path.abspath(base_path) + for file in files: + file = os.path.abspath(file) + if not os.path.isfile(file): + raise IOError("File not found: %s" % file) + # first ensure that path exists remotely + target_path = GetBaseRelativePath(path, file, base_path) + if not os.path.exists(target_path): + os.makedirs(target_path) + if verbose: + print "Copying " + file + " to " + target_path + shutil.copy(file, target_path) + def WriteProperties(files, properties_file, url_properties, package): properties = url_properties for file in files: @@ -280,16 +304,30 @@ if __name__ == '__main__': if not options.properties_file: print "You must specify a --properties-file" sys.exit(1) + + if host == "localhost": + if upload_to_temp_dir: + print "Cannot use UPLOAD_TO_TEMP with UPLOAD_HOST=localhost" + sys.exit(1) + if post_upload_command: + # POST_UPLOAD_COMMAND is difficult to extract from the mozharness + # scripts, so just ignore it until it's no longer used anywhere + print "Ignoring POST_UPLOAD_COMMAND with UPLOAD_HOST=localhost" + try: - url_properties = UploadFiles(user, host, path, args, base_path=options.base_path, - port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir, - post_upload_command=post_upload_command, - package=options.package, - verbose=True) - WriteProperties(args, options.properties_file, url_properties, options.package) + if host == "localhost": + CopyFilesLocally(path, args, base_path=options.base_path, + package=options.package, + verbose=True) + else: + + url_properties = UploadFiles(user, host, path, args, + base_path=options.base_path, port=port, ssh_key=key, + upload_to_temp_dir=upload_to_temp_dir, + post_upload_command=post_upload_command, + package=options.package, verbose=True) + + WriteProperties(args, options.properties_file, url_properties, options.package) except IOError, (strerror): print strerror sys.exit(1) - except Exception, (err): - print err - sys.exit(2) diff --git a/testing/mozharness/configs/builds/build_pool_specifics.py b/testing/mozharness/configs/builds/build_pool_specifics.py index 656f3780a0bb..81a8d72dc67b 100644 --- a/testing/mozharness/configs/builds/build_pool_specifics.py +++ b/testing/mozharness/configs/builds/build_pool_specifics.py @@ -38,9 +38,13 @@ config = { "taskcluster": { 'graph_server': 'graphs.mozilla.org', 'symbol_server_host': "symbolpush.mozilla.org", - 'stage_server': 'stage.mozilla.org', + 'stage_server': 'ignored', # use the relengapi proxy to talk to tooltool "tooltool_servers": ['http://relengapi/tooltool/'], "tooltool_url": 'http://relengapi/tooltool/', + 'upload_env': { + 'UPLOAD_HOST': 'localhost', + 'UPLOAD_PATH': '/home/worker/artifacts', + }, }, } diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index 0c14f8517349..d2348e85d494 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -826,15 +826,18 @@ or run without that action (ie: --no-{action})" mach_env = {} if c.get('upload_env'): mach_env.update(c['upload_env']) - mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % { - 'stage_server': c['stage_server'] - } - mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % { - 'stage_username': c['stage_username'] - } - mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % { - 'stage_ssh_key': c['stage_ssh_key'] - } + if 'UPLOAD_HOST' in mach_env: + mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % { + 'stage_server': c['stage_server'] + } + if 'UPLOAD_USER' in mach_env: + mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % { + 'stage_username': c['stage_username'] + } + if 'UPLOAD_SSH_KEY' in mach_env: + mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % { + 'stage_ssh_key': c['stage_ssh_key'] + } if self.query_is_nightly(): mach_env['LATEST_MAR_DIR'] = c['latest_mar_dir'] % { diff --git a/testing/taskcluster/scripts/builder/build-linux.sh b/testing/taskcluster/scripts/builder/build-linux.sh index beeb187926f2..63e4bf523ad0 100644 --- a/testing/taskcluster/scripts/builder/build-linux.sh +++ b/testing/taskcluster/scripts/builder/build-linux.sh @@ -25,25 +25,17 @@ echo "running as" $(id) : WORKSPACE ${WORKSPACE:=/home/worker/workspace} -# files to be "uploaded" (moved to ~/artifacts) from obj-firefox/dist -: DIST_UPLOADS ${DIST_UPLOADS:=""} -# files which will be be prefixed with target before being sent to artifacts -# e.g. DIST_TARGET_UPLOADS="a.zip" runs mv v2.0.a.zip mv artifacts/target.a.zip -: DIST_TARGET_UPLOADS ${DIST_TARGET_UPLOADS:=""} - set -v -# Don't run the upload step; this is passed through mozharness to mach. Once -# the mozharness scripts are not run in Buildbot anymore, this can be moved to -# Mozharness (or the upload step removed from mach entirely) -export MOZ_AUTOMATION_UPLOAD=0 - export MOZ_CRASHREPORTER_NO_REPORT=1 export MOZ_OBJDIR=obj-firefox export MOZ_SYMBOLS_EXTRA_BUILDID=linux64 -export POST_SYMBOL_UPLOAD_CMD=/usr/local/bin/post-symbol-upload.py export TINDERBOX_OUTPUT=1 +# use "simple" package names so that they can be hard-coded in the task's +# extras.locations +export MOZ_SIMPLE_PACKAGE_NAME=target + # Ensure that in tree libraries can be found export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64 @@ -136,20 +128,3 @@ python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \ --no-action=generate-build-stats \ --branch=${MH_BRANCH} \ --build-pool=${MH_BUILD_POOL} - -mkdir -p /home/worker/artifacts - -# upload auxiliary files -cd $WORKSPACE/build/src/obj-firefox/dist - -for file in $DIST_UPLOADS -do - mv $file $HOME/artifacts/$file -done - -# Discard version numbers from packaged files, they just make it hard to write -# the right filename in the task payload where artifacts are declared -for file in $DIST_TARGET_UPLOADS -do - mv *.$file $HOME/artifacts/target.$file -done diff --git a/testing/taskcluster/tasks/builds/linux64_clobber.yml b/testing/taskcluster/tasks/builds/linux64_clobber.yml index 0a567b26c32c..b05c267f3c71 100644 --- a/testing/taskcluster/tasks/builds/linux64_clobber.yml +++ b/testing/taskcluster/tasks/builds/linux64_clobber.yml @@ -30,8 +30,6 @@ task: # image paths TOOLTOOL_CACHE: '/home/worker/tooltool-cache' NEED_XVFB: true - DIST_UPLOADS: 'jsshell-linux-x86_64.zip' - DIST_TARGET_UPLOADS: 'x-test.linux-x86_64.tar.bz2 linux-x86_64.tar.bz2 linux-x86_64.json tests.zip crashreporter-symbols.zip' maxRunTime: 36000 @@ -47,7 +45,7 @@ task: platform: linux64 # Rather then enforcing particular conventions we require that all build # tasks provide the "build" extra field to specify where the build and tests - # files are located. + # files are located, relative to the task's artifacts URL locations: - build: 'public/build/target.linux-x86_64.tar.bz2' + build: 'public/build/target.tar.bz2' tests: 'public/build/target.tests.zip' diff --git a/testing/taskcluster/tasks/builds/opt_macosx64.yml b/testing/taskcluster/tasks/builds/opt_macosx64.yml index c12c4ac9a230..14d5de47ee97 100644 --- a/testing/taskcluster/tasks/builds/opt_macosx64.yml +++ b/testing/taskcluster/tasks/builds/opt_macosx64.yml @@ -39,8 +39,6 @@ task: CPLUS_INCLUDE_PATH: "" # image paths TOOLTOOL_CACHE: '/home/worker/tooltool-cache' - DIST_UPLOADS: 'jsshell-mac64.zip' - DIST_TARGET_UPLOADS: 'mac64.dmg mac64.json tests.zip crashreporter-symbols.zip' maxRunTime: 36000