bug 1423081 - add release partials support. r=callek

MozReview-Commit-ID: GCFevQVeIyy

--HG--
extra : rebase_source : 68b23ad67ee1c6fe04faacaeb2c053fb4d45ec05
extra : histedit_source : 4d99add68b706bf13500aa999dccd2af8ea5b667

taskcluster/ci/partials-signing/kind.yml

@@ -11,3 +11,6 @@ transforms:
 
 kind-dependencies:
   - partials
+
+job-template:
+  shipping-phase: promote

taskcluster/ci/partials/kind.yml

@@ -5,19 +5,27 @@
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
-  - taskgraph.transforms.name_sanity:transforms
-  - taskgraph.transforms.partials:transforms
-  - taskgraph.transforms.task:transforms
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.partials:transforms
+   - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-  - repackage-signing
+   - repackage-signing
 
 only-for-attributes:
-  - nightly
+   - nightly
 
 only-for-build-platforms:
-  - macosx64-nightly/opt
-  - win32-nightly/opt
-  - win64-nightly/opt
-  - linux-nightly/opt
-  - linux64-nightly/opt
+   - macosx64-nightly/opt
+   - macosx64-devedition-nightly/opt
+   - win32-nightly/opt
+   - win32-devedition-nightly/opt
+   - win64-nightly/opt
+   - win64-devedition-nightly/opt
+   - linux-nightly/opt
+   - linux-devedition-nightly/opt
+   - linux64-nightly/opt
+   - linux64-devedition-nightly/opt
+
+job-template:
+   shipping-phase: promote

taskcluster/taskgraph/transforms/partials.py

@@ -92,7 +92,7 @@ def make_task_description(config, jobs):
             'target.complete.mar'
         )
         for build in sorted(builds):
-            extra['funsize']['partials'].append({
+            partial_info = {
                 'locale': build_locale,
                 'from_mar': builds[build]['mar_url'],
                 'to_mar': {'task-reference': artifact_path},
@@ -100,12 +100,30 @@ def make_task_description(config, jobs):
                 'branch': config.params['project'],
                 'update_number': update_number,
                 'dest_mar': build,
-            })
+            }
+            if 'product' in builds[build]:
+                partial_info['product'] = builds[build]['product']
+            if 'previousVersion' in builds[build]:
+                partial_info['previousVersion'] = builds[build]['previousVersion']
+            if 'previousBuildNumber' in builds[build]:
+                partial_info['previousBuildNumber'] = builds[build]['previousBuildNumber']
+            extra['funsize']['partials'].append(partial_info)
             update_number += 1
 
         cot = extra.setdefault('chainOfTrust', {})
         cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
 
+        mar_channel_id = None
+        if config.params['project'] == 'mozilla-beta':
+            if 'devedition' in label:
+                mar_channel_id = 'firefox-mozilla-aurora'
+            else:
+                mar_channel_id = 'firefox-mozilla-beta'
+        elif config.params['project'] == 'mozilla-release':
+            mar_channel_id = 'firefox-mozilla-release'
+        elif 'esr' in config.params['project']:
+            mar_channel_id = 'firefox-mozilla-esr'
+
         worker = {
             'artifacts': _generate_task_output_files(builds.keys(), locale),
             'implementation': 'docker-worker',
@@ -120,6 +138,8 @@ def make_task_description(config, jobs):
                 'DATADOG_API_SECRET': 'project/releng/gecko/build/level-3/datadog-api-key'
             }
         }
+        if mar_channel_id:
+            worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id
 
         level = config.params['level']
 

taskcluster/taskgraph/util/partials.py

@@ -45,6 +45,20 @@ BALROG_PLATFORM_MAP = {
     ]
 }
 
+FTP_PLATFORM_MAP = {
+    "Darwin_x86-gcc3": "mac",
+    "Darwin_x86-gcc3-u-i386-x86_64": "mac",
+    "Darwin_x86_64-gcc3": "mac",
+    "Darwin_x86_64-gcc3-u-i386-x86_64": "mac",
+    "Linux_x86-gcc3": "linux-i686",
+    "Linux_x86_64-gcc3": "linux-x86_64",
+    "WINNT_x86-msvc": "win32",
+    "WINNT_x86-msvc-x64": "win32",
+    "WINNT_x86-msvc-x86": "win32",
+    "WINNT_x86_64-msvc": "win64",
+    "WINNT_x86_64-msvc-x64": "win64",
+}
+
 
 def get_balrog_platform_name(platform):
     """Convert build platform names into balrog platform names"""
@@ -76,8 +90,16 @@ def get_partials_artifacts(release_history, platform, locale):
 
 def get_partials_artifact_map(release_history, platform, locale):
     platform = _sanitize_platform(platform)
-    return {k: release_history[platform][locale][k]['buildid']
-            for k in release_history.get(platform, {}).get(locale, {})}
+
+    artifact_map = {}
+    for k in release_history.get(platform, {}).get(locale, {}):
+        details = release_history[platform][locale][k]
+        attributes = ('buildid',
+                      'previousBuildNumber',
+                      'previousVersion')
+        artifact_map[k] = {attr: details[attr] for attr in attributes
+                           if attr in details}
+    return artifact_map
 
 
 def _retry_on_http_errors(url, verify, params, errors):
@@ -133,7 +155,25 @@ def get_release_builds(release):
     return req.json()
 
 
-def populate_release_history(product, branch, maxbuilds=4, maxsearch=10):
+def find_localtest(fileUrls):
+    for channel in fileUrls:
+        if "-localtest" in channel:
+            return channel
+
+
+def populate_release_history(product, branch, maxbuilds=4, maxsearch=10,
+                             partial_updates=None):
+    # Assuming we are using release branches when we know the list of previous
+    # releases in advance
+    if partial_updates:
+        return _populate_release_history(
+            product, branch, partial_updates=partial_updates)
+    else:
+        return _populate_nightly_history(
+            product, branch, maxbuilds=maxbuilds, maxsearch=maxsearch)
+
+
+def _populate_nightly_history(product, branch, maxbuilds=4, maxsearch=10):
     """Find relevant releases in Balrog
     Not all releases have all platforms and locales, due
     to Taskcluster migration.
@@ -194,3 +234,38 @@ def populate_release_history(product, branch, maxbuilds=4, maxsearch=10):
                     'mar_url': url,
                 }
     return builds
+
+
+def _populate_release_history(product, branch, partial_updates):
+    builds = dict()
+    for version, release in partial_updates.iteritems():
+        prev_release_blob = '{product}-{version}-build{build_number}'.format(
+            product=product, version=version, build_number=release['buildNumber']
+        )
+        partial_mar_key = 'target-{version}.partial.mar'.format(version=version)
+        history = get_release_builds(prev_release_blob)
+        # use one of the localtest channels to avoid relying on bouncer
+        localtest = find_localtest(history['fileUrls'])
+        url_pattern = history['fileUrls'][localtest]['completes']['*']
+
+        for platform in history['platforms']:
+            if 'alias' in history['platforms'][platform]:
+                continue
+            if platform not in builds:
+                builds[platform] = dict()
+            for locale in history['platforms'][platform]['locales']:
+                if locale not in builds[platform]:
+                    builds[platform][locale] = dict()
+                buildid = history['platforms'][platform]['locales'][locale]['buildID']
+                url = url_pattern.replace(
+                    '%OS_FTP%', FTP_PLATFORM_MAP[platform]).replace(
+                    '%LOCALE%', locale
+                )
+                builds[platform][locale][partial_mar_key] = {
+                    'buildid': buildid,
+                    'mar_url': url,
+                    'previousVersion': version,
+                    'previousBuildNumber': release['buildNumber'],
+                    'product': product,
+                }
+    return builds