Bug 1198179: in taskcluster, have 'make upload' just copy; r=ted.mielczarek

--HG--
extra : commitid : C1f3kiibCXn
extra : rebase_source : ed715e77014b91fb19c7a6bc7a0a04134f1e69cc
extra : amend_source : 46abf6b9bfc0f7993da9814d8f6e3959379c434f

build/upload.py

@@ -15,6 +15,10 @@
 # If UPLOAD_HOST and UPLOAD_USER are not set, this script will simply write out
 # the properties file.
 #
+# If UPLOAD_HOST is "localhost", then files are simply copied to UPLOAD_PATH.
+# In this case, UPLOAD_TO_TEMP and POST_UPLOAD_CMD are not supported, and no
+# properties are written out.
+#
 # And will use the following optional environment variables if set:
 # UPLOAD_SSH_KEY : path to a ssh private key to use
 # UPLOAD_PORT    : port to use for ssh
@@ -32,6 +36,7 @@ import re
 import json
 import errno
 import hashlib
+import shutil
 from optparse import OptionParser
 from subprocess import check_call, check_output, STDOUT
 import redo
@@ -106,7 +111,7 @@ def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None):
 
     raise Exception("Command %s returned non-zero exit code" % cmdline)
 
-def GetRemotePath(path, local_file, base_path):
+def GetBaseRelativePath(path, local_file, base_path):
     """Given a remote path to upload to, a full path to a local file, and an
     optional full path that is a base path of the local file, construct the
     full remote path to place the file in. If base_path is not None, include
@@ -214,7 +219,7 @@ def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None,
             if not os.path.isfile(file):
                 raise IOError("File not found: %s" % file)
             # first ensure that path exists remotely
-            remote_path = GetRemotePath(path, file, base_path)
+            remote_path = GetBaseRelativePath(path, file, base_path)
             DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
             if verbose:
                 print "Uploading " + file
@@ -236,6 +241,25 @@ def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None,
         print "Upload complete"
     return properties
 
+def CopyFilesLocally(path, files, verbose=False, base_path=None, package=None):
+    """Copy each file in the list of files to `path`.  The `base_path` argument is treated
+    as it is by UploadFiles."""
+    if not path.endswith("/"):
+        path += "/"
+    if base_path is not None:
+        base_path = os.path.abspath(base_path)
+    for file in files:
+        file = os.path.abspath(file)
+        if not os.path.isfile(file):
+            raise IOError("File not found: %s" % file)
+        # first ensure that path exists remotely
+        target_path = GetBaseRelativePath(path, file, base_path)
+        if not os.path.exists(target_path):
+            os.makedirs(target_path)
+        if verbose:
+            print "Copying " + file + " to " + target_path
+        shutil.copy(file, target_path)
+
 def WriteProperties(files, properties_file, url_properties, package):
     properties = url_properties
     for file in files:
@@ -280,16 +304,30 @@ if __name__ == '__main__':
     if not options.properties_file:
         print "You must specify a --properties-file"
         sys.exit(1)
+
+    if host == "localhost":
+        if upload_to_temp_dir:
+            print "Cannot use UPLOAD_TO_TEMP with UPLOAD_HOST=localhost"
+            sys.exit(1)
+        if post_upload_command:
+            # POST_UPLOAD_COMMAND is difficult to extract from the mozharness
+            # scripts, so just ignore it until it's no longer used anywhere
+            print "Ignoring POST_UPLOAD_COMMAND with UPLOAD_HOST=localhost"
+
     try:
-        url_properties = UploadFiles(user, host, path, args, base_path=options.base_path,
-                                     port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
-                                     post_upload_command=post_upload_command,
-                                     package=options.package,
-                                     verbose=True)
-        WriteProperties(args, options.properties_file, url_properties, options.package)
+        if host == "localhost":
+            CopyFilesLocally(path, args, base_path=options.base_path,
+                             package=options.package,
+                             verbose=True)
+        else:
+
+            url_properties = UploadFiles(user, host, path, args,
+                                         base_path=options.base_path, port=port, ssh_key=key,
+                                         upload_to_temp_dir=upload_to_temp_dir,
+                                         post_upload_command=post_upload_command,
+                                         package=options.package, verbose=True)
+
+            WriteProperties(args, options.properties_file, url_properties, options.package)
     except IOError, (strerror):
         print strerror
         sys.exit(1)
-    except Exception, (err):
-        print err
-        sys.exit(2)

testing/mozharness/configs/builds/build_pool_specifics.py

@@ -38,9 +38,13 @@ config = {
     "taskcluster": {
         'graph_server': 'graphs.mozilla.org',
         'symbol_server_host': "symbolpush.mozilla.org",
-        'stage_server': 'stage.mozilla.org',
+        'stage_server': 'ignored',
         # use the relengapi proxy to talk to tooltool
         "tooltool_servers": ['http://relengapi/tooltool/'],
         "tooltool_url": 'http://relengapi/tooltool/',
+        'upload_env': {
+            'UPLOAD_HOST': 'localhost',
+            'UPLOAD_PATH': '/home/worker/artifacts',
+        },
     },
 }

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -826,15 +826,18 @@ or run without that action (ie: --no-{action})"
         mach_env = {}
         if c.get('upload_env'):
             mach_env.update(c['upload_env'])
-            mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % {
-                'stage_server': c['stage_server']
-            }
-            mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % {
-                'stage_username': c['stage_username']
-            }
-            mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % {
-                'stage_ssh_key': c['stage_ssh_key']
-            }
+            if 'UPLOAD_HOST' in mach_env:
+                mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % {
+                    'stage_server': c['stage_server']
+                }
+            if 'UPLOAD_USER' in mach_env:
+                mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % {
+                    'stage_username': c['stage_username']
+                }
+            if 'UPLOAD_SSH_KEY' in mach_env:
+                mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % {
+                    'stage_ssh_key': c['stage_ssh_key']
+                }
 
         if self.query_is_nightly():
             mach_env['LATEST_MAR_DIR'] = c['latest_mar_dir'] % {

testing/taskcluster/scripts/builder/build-linux.sh

@@ -25,25 +25,17 @@ echo "running as" $(id)
 
 : WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
 
-# files to be "uploaded" (moved to ~/artifacts) from obj-firefox/dist
-: DIST_UPLOADS                  ${DIST_UPLOADS:=""}
-# files which will be be prefixed with target before being sent to artifacts
-# e.g. DIST_TARGET_UPLOADS="a.zip" runs mv v2.0.a.zip mv artifacts/target.a.zip
-: DIST_TARGET_UPLOADS           ${DIST_TARGET_UPLOADS:=""}
-
 set -v
 
-# Don't run the upload step; this is passed through mozharness to mach.  Once
-# the mozharness scripts are not run in Buildbot anymore, this can be moved to
-# Mozharness (or the upload step removed from mach entirely)
-export MOZ_AUTOMATION_UPLOAD=0
-
 export MOZ_CRASHREPORTER_NO_REPORT=1
 export MOZ_OBJDIR=obj-firefox
 export MOZ_SYMBOLS_EXTRA_BUILDID=linux64
-export POST_SYMBOL_UPLOAD_CMD=/usr/local/bin/post-symbol-upload.py
 export TINDERBOX_OUTPUT=1
 
+# use "simple" package names so that they can be hard-coded in the task's
+# extras.locations
+export MOZ_SIMPLE_PACKAGE_NAME=target
+
 # Ensure that in tree libraries can be found
 export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64
 
@@ -136,20 +128,3 @@ python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
   --no-action=generate-build-stats \
   --branch=${MH_BRANCH} \
   --build-pool=${MH_BUILD_POOL}
-
-mkdir -p /home/worker/artifacts
-
-# upload auxiliary files
-cd $WORKSPACE/build/src/obj-firefox/dist
-
-for file in $DIST_UPLOADS
-do
-    mv $file $HOME/artifacts/$file
-done
-
-# Discard version numbers from packaged files, they just make it hard to write
-# the right filename in the task payload where artifacts are declared
-for file in $DIST_TARGET_UPLOADS
-do
-    mv *.$file $HOME/artifacts/target.$file
-done

testing/taskcluster/tasks/builds/linux64_clobber.yml

@@ -30,8 +30,6 @@ task:
       # image paths
       TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
       NEED_XVFB: true
-      DIST_UPLOADS: 'jsshell-linux-x86_64.zip'
-      DIST_TARGET_UPLOADS: 'x-test.linux-x86_64.tar.bz2 linux-x86_64.tar.bz2 linux-x86_64.json tests.zip crashreporter-symbols.zip'
 
     maxRunTime: 36000
 
@@ -47,7 +45,7 @@ task:
         platform: linux64
     # Rather then enforcing particular conventions we require that all build
     # tasks provide the "build" extra field to specify where the build and tests
-    # files are located.
+    # files are located, relative to the task's artifacts URL
     locations:
-      build: 'public/build/target.linux-x86_64.tar.bz2'
+      build: 'public/build/target.tar.bz2'
       tests: 'public/build/target.tests.zip'

testing/taskcluster/tasks/builds/opt_macosx64.yml

@@ -39,8 +39,6 @@ task:
       CPLUS_INCLUDE_PATH: ""
       # image paths
       TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
-      DIST_UPLOADS: 'jsshell-mac64.zip'
-      DIST_TARGET_UPLOADS: 'mac64.dmg mac64.json tests.zip crashreporter-symbols.zip'
 
     maxRunTime: 36000