bug 1398799: mozharness script to create update verify configs without relying on patcher configs. r=nthomas

python/moz.build

@@ -29,6 +29,9 @@ with Files('mozversioncontrol/**'):
 with Files('l10n/**'):
     BUG_COMPONENT = ('Core', 'Localization')
 
+with Files('mozrelease/**'):
+    BUG_COMPONENT = ('Release Engineering', 'Release Automation')
+
 with Files('mach_commands.py'):
     BUG_COMPONENT = ('Testing', 'Python Test')
 

python/mozrelease/mozrelease/chunking.py

@@ -0,0 +1,24 @@
+from __future__ import absolute_import
+
+from copy import copy
+
+
+class ChunkingError(Exception):
+    pass
+
+
+def getChunk(things, chunks, thisChunk):
+    if thisChunk > chunks:
+        raise ChunkingError("thisChunk (%d) is greater than total chunks (%d)" %
+                           (thisChunk, chunks))
+    possibleThings = copy(things)
+    nThings = len(possibleThings)
+    for c in range(1, chunks + 1):
+        n = nThings / chunks
+        # If our things aren't evenly divisible by the number of chunks
+        # we need to append one more onto some of them
+        if c <= (nThings % chunks):
+            n += 1
+        if c == thisChunk:
+            return possibleThings[0:n]
+        del possibleThings[0:n]

python/mozrelease/mozrelease/l10n.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import
+
+from .platforms import shippedLocales2ftp
+
+
+def getPlatformLocales(shipped_locales, platform):
+    platform_locales = []
+    for line in shipped_locales.splitlines():
+        entry = line.strip().split()
+        locale = entry[0]
+        if len(entry) > 1:
+            for sl_platform in entry[1:]:
+                if platform in shippedLocales2ftp(sl_platform):
+                    platform_locales.append(locale)
+        else:
+            platform_locales.append(locale)
+    return platform_locales

python/mozrelease/mozrelease/paths.py

@@ -0,0 +1,63 @@
+from __future__ import absolute_import
+
+from urlparse import urlunsplit
+
+product_ftp_map = {
+    "fennec": "mobile",
+}
+
+def product2ftp(product):
+    return product_ftp_map.get(product, product)
+
+
+def getCandidatesDir(product, version, buildNumber, protocol=None, server=None):
+    if protocol:
+        assert server is not None, "server is required with protocol"
+
+    product = product2ftp(product)
+    directory = "/{}/candidates/{}-candidates/build{}".format(
+        product, str(version), str(buildNumber)
+    )
+
+    if protocol:
+        return urlunsplit((protocol, server, directory, None, None))
+    else:
+        return directory
+
+
+def getReleasesDir(product, version=None, protocol=None, server=None):
+    if protocol:
+        assert server is not None, "server is required with protocol"
+
+    directory = "/{}/releases".format(product)
+    if version:
+        directory = "{}/{}".format(directory, version)
+
+    if protocol:
+        return urlunsplit((protocol, server, directory, None, None))
+    else:
+        return directory
+
+
+def getReleaseInstallerPath(productName, brandName, version, platform,
+                            locale='en-US'):
+    if productName not in ('fennec',):
+        if platform.startswith('linux'):
+            return '/'.join([p.strip('/') for p in [
+                platform, locale, '%s-%s.tar.bz2' % (productName, version)]])
+        elif 'mac' in platform:
+            return '/'.join([p.strip('/') for p in [
+                platform, locale, '%s %s.dmg' % (brandName, version)]])
+        elif platform.startswith('win'):
+            return '/'.join([p.strip('/') for p in [
+                platform, locale, '%s Setup %s.exe' % (brandName, version)]])
+        else:
+            raise "Unsupported platform"
+    else:
+        if platform.startswith('android'):
+            filename = '%s-%s.%s.android-arm.apk' % (
+                productName, version, locale)
+            return '/'.join([p.strip('/') for p in [
+                platform, locale, filename]])
+        else:
+            raise "Unsupported platform"

python/mozrelease/mozrelease/platforms.py

@@ -0,0 +1,60 @@
+from __future__ import absolute_import
+
+# ftp -> update platform map
+update_platform_map = {
+    "android": ["Android_arm-eabi-gcc3"],
+    "android-api-11": ["Android_arm-eabi-gcc3"],
+    "android-api-15": ["Android_arm-eabi-gcc3"],
+    "android-api-15-old-id": ["Android_arm-eabi-gcc3"],
+    "android-api-16": ["Android_arm-eabi-gcc3"],
+    "android-api-16-old-id": ["Android_arm-eabi-gcc3"],
+    "android-x86": ["Android_x86-gcc3"],
+    "android-x86-old-id": ["Android_x86-gcc3"],
+    "android-aarch64": ["Android_aarch64-gcc3"],
+    "linux-i686": ["Linux_x86-gcc3"],
+    "linux-x86_64": ["Linux_x86_64-gcc3"],
+    "mac": ["Darwin_x86_64-gcc3-u-i386-x86_64",  # The main platofrm
+            "Darwin_x86-gcc3-u-i386-x86_64",
+            # We don"t ship builds with these build targets, but some users
+            # modify their builds in a way that has them report like these.
+            # See bug 1071576 for details.
+            "Darwin_x86-gcc3", "Darwin_x86_64-gcc3"],
+    "win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"],
+    "win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"],
+}
+
+# ftp -> shipped locales map
+sl_platform_map = {
+    "linux-i686": "linux",
+    "linux-x86_64": "linux",
+    "mac": "osx",
+    "win32": "win32",
+    "win64": "win64",
+}
+
+# ftp -> info file platform map
+info_file_platform_map = {
+    "linux-i686": "linux",
+    "linux-x86_64": "linux64",
+    "mac": "macosx64",
+    "win32": "win32",
+    "win64": "win64",
+}
+
+def ftp2updatePlatforms(platform):
+    return update_platform_map.get(platform, platform)
+
+def ftp2shippedLocales(platform):
+    return sl_platform_map.get(platform, platform)
+
+def shippedLocales2ftp(platform):
+    matches = []
+    try:
+        [matches.append(
+            k) for k, v in sl_platform_map.iteritems() if v == platform][0]
+        return matches
+    except IndexError:
+        return [platform]
+
+def ftp2infoFile(platform):
+    return info_file_platform_map.get(platform, platform)

python/mozrelease/mozrelease/update_verify.py

@@ -0,0 +1,190 @@
+from __future__ import absolute_import
+
+import os
+import re
+
+from .chunking import getChunk
+
+
+class UpdateVerifyError(Exception):
+    pass
+
+
+class UpdateVerifyConfig(object):
+    comment_regex = re.compile("^#")
+    key_write_order = ("release", "product", "platform", "build_id", "locales",
+                       "channel", "patch_types", "from", "aus_server",
+                       "ftp_server_from", "ftp_server_to", "to",
+                       "mar_channel_IDs", "to_build_id", "to_display_version",
+                       "to_app_version", "updater_package")
+    global_keys = ("product", "channel", "aus_server", "to", "to_build_id",
+                   "to_display_version", "to_app_version")
+    release_keys = ("release", "build_id", "locales", "patch_types", "from",
+                    "ftp_server_from", "ftp_server_to", "mar_channel_IDs",
+                    "platform", "updater_package")
+    first_only_keys = ("from", "aus_server", "to", "to_build_id",
+                       "to_display_version", "to_app_version")
+    compare_attrs = global_keys + ("releases",)
+
+    def __init__(self, product=None, channel=None,
+                 aus_server=None, to=None, to_build_id=None,
+                 to_display_version=None, to_app_version=None):
+        self.product = product
+        self.channel = channel
+        self.aus_server = aus_server
+        self.to = to
+        self.to_build_id = to_build_id
+        self.to_display_version = to_display_version
+        self.to_app_version = to_app_version
+        self.releases = []
+
+    def __eq__(self, other):
+        self_list = [getattr(self, attr) for attr in self.compare_attrs]
+        other_list = [getattr(other, attr) for attr in self.compare_attrs]
+        return self_list == other_list
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def _parseLine(self, line):
+        entry = {}
+        items = re.findall("\w+=[\"'][^\"']*[\"']", line)
+        for i in items:
+            m = re.search(
+                "(?P<key>\w+)=[\"'](?P<value>.+)[\"']", i).groupdict()
+            if m["key"] not in self.global_keys and m["key"] not in self.release_keys:
+                raise UpdateVerifyError(
+                    "Unknown key '%s' found on line:\n%s" % (m["key"], line))
+            if m["key"] in entry:
+                raise UpdateVerifyError("Multiple values found for key '%s' on line:\n%s" % (m["key"], line))
+            entry[m["key"]] = m["value"]
+        if not entry:
+            raise UpdateVerifyError("No parseable data in line '%s'" % line)
+        return entry
+
+    def _addEntry(self, entry, first):
+        releaseKeys = {}
+        for k, v in entry.items():
+            if k in self.global_keys:
+                setattr(self, k, entry[k])
+            elif k in self.release_keys:
+                # "from" is reserved in Python
+                if k == "from":
+                    releaseKeys["from_path"] = v
+                else:
+                    releaseKeys[k] = v
+        self.addRelease(**releaseKeys)
+
+    def read(self, config):
+        f = open(config)
+        # Only the first non-comment line of an update verify config should
+        # have a "from" and"ausServer". Ignore any subsequent lines with them.
+        first = True
+        for line in f.readlines():
+            # Skip comment lines
+            if self.comment_regex.search(line):
+                continue
+            self._addEntry(self._parseLine(line), first)
+            first = False
+
+    def write(self, fh):
+        first = True
+        for releaseInfo in self.releases:
+            for key in self.key_write_order:
+                if key in self.global_keys and (first or key not in self.first_only_keys):
+                    value = getattr(self, key)
+                elif key in self.release_keys:
+                    value = releaseInfo[key]
+                else:
+                    value = None
+                if value is not None:
+                    fh.write(key)
+                    fh.write("=")
+                    if isinstance(value, (list, tuple)):
+                        fh.write('"%s" ' % " ".join(value))
+                    else:
+                        fh.write('"%s" ' % str(value))
+            # Rewind one character to avoid having a trailing space
+            fh.seek(-1, os.SEEK_CUR)
+            fh.write("\n")
+            first = False
+
+    def addRelease(self, release=None, build_id=None, locales=[],
+                   patch_types=['complete'], from_path=None,
+                   ftp_server_from=None, ftp_server_to=None,
+                   mar_channel_IDs=None, platform=None, updater_package=None):
+        """Locales and patch_types can be passed as either a string or a list.
+           If a string is passed, they will be converted to a list for internal
+           storage"""
+        if self.getRelease(build_id, from_path):
+            raise UpdateVerifyError("Couldn't add release identified by build_id '%s' and from_path '%s': already exists in config" % (build_id, from_path))
+        if isinstance(locales, basestring):
+            locales = sorted(list(locales.split()))
+        if isinstance(patch_types, basestring):
+            patch_types = list(patch_types.split())
+        self.releases.append({
+            "release": release,
+            "build_id": build_id,
+            "locales": locales,
+            "patch_types": patch_types,
+            "from": from_path,
+            "ftp_server_from": ftp_server_from,
+            "ftp_server_to": ftp_server_to,
+            "mar_channel_IDs": mar_channel_IDs,
+            "platform": platform,
+            "updater_package": updater_package,
+        })
+
+    def addLocaleToRelease(self, build_id, locale, from_path=None):
+        r = self.getRelease(build_id, from_path)
+        if not r:
+            raise UpdateVerifyError("Couldn't add '%s' to release identified by build_id '%s' and from_path '%s': '%s' doesn't exist in this config." % (locale, build_id, from_path, build_id))
+        r["locales"].append(locale)
+        r["locales"] = sorted(r["locales"])
+
+    def getRelease(self, build_id, from_path):
+        for r in self.releases:
+            if r["build_id"] == build_id and r["from"] == from_path:
+                return r
+        return {}
+
+    def getFullReleaseTests(self):
+        return [r for r in self.releases if r["from"] is not None]
+
+    def getQuickReleaseTests(self):
+        return [r for r in self.releases if r["from"] is None]
+
+    def getChunk(self, chunks, thisChunk):
+        fullTests = []
+        quickTests = []
+        for test in self.getFullReleaseTests():
+            for locale in test["locales"]:
+                fullTests.append([test["build_id"], locale, test["from"]])
+        for test in self.getQuickReleaseTests():
+            for locale in test["locales"]:
+                quickTests.append([test["build_id"], locale, test["from"]])
+        allTests = getChunk(fullTests, chunks, thisChunk)
+        allTests.extend(getChunk(quickTests, chunks, thisChunk))
+
+        newConfig = UpdateVerifyConfig(self.product, self.channel,
+                                       self.aus_server, self.to,
+                                       self.to_build_id,
+                                       self.to_display_version,
+                                       self.to_app_version)
+        for t in allTests:
+            build_id, locale, from_path = t
+            if from_path == "None":
+                from_path = None
+            r = self.getRelease(build_id, from_path)
+            try:
+                newConfig.addRelease(r["release"], build_id, locales=[],
+                                     ftp_server_from=r["ftp_server_from"],
+                                     ftp_server_to=r["ftp_server_to"],
+                                     patch_types=r["patch_types"], from_path=from_path,
+                                     mar_channel_IDs=r["mar_channel_IDs"],
+                                     platform=r["platform"],
+                                     updater_package=r["updater_package"])
+            except UpdateVerifyError:
+                pass
+            newConfig.addLocaleToRelease(build_id, locale, from_path)
+        return newConfig

python/mozrelease/mozrelease/versions.py

@@ -0,0 +1,44 @@
+from __future__ import absolute_import
+
+from distutils.version import StrictVersion
+import re
+
+
+class ModernMozillaVersion(StrictVersion):
+    """A version class that is slightly less restrictive than StrictVersion.
+       Instead of just allowing "a" or "b" as prerelease tags, it allows any
+       alpha. This allows us to support the once-shipped "3.6.3plugin1" and
+       similar versions."""
+    version_re = re.compile(r"""^(\d+) \. (\d+) (\. (\d+))?
+                                ([a-zA-Z]+(\d+))?$""", re.VERBOSE)
+
+
+class AncientMozillaVersion(StrictVersion):
+    """A version class that is slightly less restrictive than StrictVersion.
+       Instead of just allowing "a" or "b" as prerelease tags, it allows any
+       alpha. This allows us to support the once-shipped "3.6.3plugin1" and
+       similar versions.
+       It also supports versions w.x.y.z by transmuting to w.x.z, which
+       is useful for versions like 1.5.0.x and 2.0.0.y"""
+    version_re = re.compile(r"""^(\d+) \. (\d+) \. \d (\. (\d+))
+                                ([a-zA-Z]+(\d+))?$""", re.VERBOSE)
+
+
+def MozillaVersion(version):
+    try:
+        return ModernMozillaVersion(version)
+    except ValueError:
+        pass
+    try:
+        if version.count('.') == 3:
+            return AncientMozillaVersion(version)
+    except ValueError:
+        pass
+    raise ValueError("Version number %s is invalid." % version)
+
+
+def getPrettyVersion(version):
+    version = re.sub(r'a([0-9]+)$', r' Alpha \1', version)
+    version = re.sub(r'b([0-9]+)$', r' Beta \1', version)
+    version = re.sub(r'rc([0-9]+)$', r' RC \1', version)
+    return version

python/mozrelease/setup.py

@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from setuptools import setup, find_packages
+
+VERSION = '0.1'
+
+setup(
+    author='Mozilla Foundation',
+    author_email='Mozilla Release Engineering',
+    name='mozrelease',
+    description='Common functionality used by Mozilla Release Automation',
+    license='MPL 2.0',
+    packages=find_packages(),
+    version=VERSION,
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Topic :: Software Development :: Build Tools',
+        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: Implementation :: CPython',
+    ],
+    keywords='mozilla',
+)

testing/mozharness/configs/internal_pypi.py

@@ -0,0 +1,7 @@
+config = {
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    'pip_index': False,
+}

testing/mozharness/scripts/release/update-verify-config-creator.py

@@ -0,0 +1,452 @@
+from distutils.version import LooseVersion
+import json
+import math
+import os
+import pprint
+import re
+import sys
+from urlparse import urljoin
+
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.log import DEBUG, INFO, FATAL
+from mozharness.base.script import BaseScript
+from mozharness.base.python import VirtualenvMixin
+
+
+def is_triangualar(x):
+    """Check if a number is triangular (0, 1, 3, 6, 10, 15, ...)
+    see: https://en.wikipedia.org/wiki/Triangular_number#Triangular_roots_and_tests_for_triangular_numbers # noqa
+
+    >>> is_triangualar(0)
+    True
+    >>> is_triangualar(1)
+    True
+    >>> is_triangualar(2)
+    False
+    >>> is_triangualar(3)
+    True
+    >>> is_triangualar(4)
+    False
+    >>> all(is_triangualar(x) for x in [0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66, 78, 91, 105])
+    True
+    >>> all(not is_triangualar(x) for x in [4, 5, 8, 9, 11, 17, 25, 29, 39, 44, 59, 61, 72, 98, 112])
+    True
+    """
+    n = (math.sqrt(8*x + 1) - 1)/2
+    return n == int(n)
+
+
+class UpdateVerifyConfigCreator(BaseScript, VirtualenvMixin):
+    config_options = [
+        [["--product"], {
+            "dest": "product",
+            "help": "Product being tested, as used in the update URL and filenames. Eg: firefox",
+        }],
+        [["--stage-product"], {
+            "dest": "stage_product",
+            "help": "Product being tested, as used in stage directories and ship it"
+                    "If not passed this is assumed to be the same as product."
+        }],
+        [["--app-name"], {
+            "dest": "app_name",
+            "help": "App name being tested. Eg: browser",
+        }],
+        [["--channel"], {
+            "dest": "channel",
+            "help": "Channel to run update verify against",
+        }],
+        [["--aus-server"], {
+            "dest": "aus_server",
+            "default": "https://aus5.mozilla.org",
+            "help": "AUS server to run update verify against",
+        }],
+        [["--to-version"], {
+            "dest": "to_version",
+            "help": "The version of the release being updated to. Eg: 59.0b5",
+        }],
+        [["--to-app-version"], {
+            "dest": "to_app_version",
+            "help": "The in-app version of the release being updated to. Eg: 59.0",
+        }],
+        [["--to-display-version"], {
+            "dest": "to_display_version",
+            "help": "The human-readable version of the release being updated to. Eg: 59.0 Beta 9",
+        }],
+        [["--to-build-number"], {
+            "dest": "to_build_number",
+            "help": "The build number of the release being updated to",
+        }],
+        [["--to-buildid"], {
+            "dest": "to_buildid",
+            "help": "The buildid of the release being updated to",
+        }],
+        [["--to-revision"], {
+            "dest": "to_revision",
+            "help": "The revision that the release being updated to was built against",
+        }],
+        [["--partial-version"], {
+            "dest": "partial_versions",
+            "action": "append",
+            "help": "A previous release version that is expected to receive a partial update. "
+                    "Eg: 59.0b4. May be specified multiple times."
+        }],
+        [["--last-watershed"], {
+            "dest": "last_watershed",
+            "help": "The earliest version to include in the update verify config. Eg: 57.0b10",
+        }],
+        [["--include-version"], {
+            "dest": "include_versions",
+            "default": [],
+            "action": "append",
+            "help": "Only include versions that match one of these regexes. "
+                    "May be passed multiple times",
+        }],
+        [["--mar-channel-id-override"], {
+            "dest": "mar_channel_id_options",
+            "default": [],
+            "action": "append",
+            "help": "A version regex and channel id string to override those versions with."
+                    "Eg: ^\\d+\\.\\d+(\\.\\d+)?$,firefox-mozilla-beta,firefox-mozilla-release "
+                    "will set accepted mar channel ids to 'firefox-mozilla-beta' and "
+                    "'firefox-mozilla-release for x.y and x.y.z versions. "
+                    "May be passed multiple times"
+        }],
+        [["--platform"], {
+            "dest": "platform",
+            "help": "The platform to generate the update verify config for, in FTP-style",
+        }],
+        [["--updater-platform"], {
+            "dest": "updater_platform",
+            "help": "The platform to run the updater on, in FTP-style."
+                    "If not specified, this is assumed to be the same as platform",
+        }],
+        [["--archive-prefix"], {
+            "dest": "archive_prefix",
+            "help": "The server/path to pull the current release from. "
+                    "Eg: https://archive.mozilla.org/pub",
+        }],
+        [["--previous-archive-prefix"], {
+            "dest": "previous_archive_prefix",
+            "help": "The server/path to pull the previous releases from"
+                    "If not specified, this is assumed to be the same as --archive-prefix"
+        }],
+        [["--repo-path"], {
+            "dest": "repo_path",
+            "help": "The repository (relative to the hg server root) that the current release was "
+                    "built from Eg: releases/mozilla-beta"
+        }],
+        [["--output-file"], {
+            "dest": "output_file",
+            "help": "Where to write the update verify config to",
+        }],
+        [["--product-details-server"], {
+            "dest": "product_details_server",
+            "default": "https://product-details.mozilla.org",
+            "help": "Product Details server to pull previous release info from. "
+                    "Using anything other than the production server is likely to "
+                    "cause issues with update verify."
+        }],
+        [["--hg-server"], {
+            "dest": "hg_server",
+            "default": "https://hg.mozilla.org",
+            "help": "Mercurial server to pull various previous and current version info from",
+        }],
+        [["--full-check-locale"], {
+            "dest": "full_check_locales",
+            "default": ["de", "en-US", "ru"],
+            "action": "append",
+            "help": "A list of locales to generate full update verify checks for",
+        }],
+    ]
+
+    def __init__(self):
+        BaseScript.__init__(
+            self,
+            config_options=self.config_options,
+            config={
+                "virtualenv_modules": [
+                    "mozrelease",
+                ],
+                "virtualenv_path": "venv",
+            },
+            all_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "gather-info",
+                "create-config",
+                "write-config",
+            ],
+            default_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "gather-info",
+                "create-config",
+                "write-config",
+            ],
+        )
+
+    def _pre_config_lock(self, rw_config):
+        super(UpdateVerifyConfigCreator, self)._pre_config_lock(rw_config)
+
+        if "updater_platform" not in self.config:
+            self.config["updater_platform"] = self.config["platform"]
+        if "stage_product" not in self.config:
+            self.config["stage_product"] = self.config["product"]
+        if "previous_archive_prefix" not in self.config:
+            self.config["previous_archive_prefix"] = self.config["archive_prefix"]
+        self.config["archive_prefix"].rstrip("/")
+        self.config["previous_archive_prefix"].rstrip("/")
+        self.config["mar_channel_id_overrides"] = {}
+        for override in self.config["mar_channel_id_options"]:
+            pattern, override_str = override.split(",", 1)
+            self.config["mar_channel_id_overrides"][pattern] = override_str
+
+    def _get_update_paths(self):
+        from mozrelease.l10n import getPlatformLocales
+        from mozrelease.paths import getCandidatesDir
+        from mozrelease.platforms import ftp2infoFile
+        from mozrelease.versions import MozillaVersion
+
+        self.update_paths = {}
+
+        ret = self._retry_download(
+            "{}/1.0/{}.json".format(
+                self.config["product_details_server"],
+                self.config["stage_product"],
+            ),
+            "WARNING",
+        )
+        releases = json.load(ret)["releases"]
+        for release_name, release_info in reversed(sorted(releases.items())):
+            product, version = release_name.split("-", 1)
+            version = version.rstrip("esr")
+            tag = "{}_{}_RELEASE".format(product.upper(), version.replace(".", "_"))
+            # Product details has a "category" for releases that we can use to
+            # determine the repo path. This will fail if any previous releases
+            # were built from a project branch - but that's not something we do
+            # at the time of writing.
+            branch = None
+            if release_info["category"] == "dev":
+                branch = "releases/mozilla-beta"
+            elif release_info["category"] == "esr":
+                branch = "releases/mozilla-esr{}".format(version[:2])
+            elif release_info["category"] in ("major", "stability"):
+                branch = "releases/mozilla-release"
+            if not branch:
+                raise Exception("Cannot determine branch, cannot continue!")
+
+            # Exclude any releases that don't match one of our include version
+            # regexes. This is generally to avoid including versions from other
+            # channels. Eg: including betas when testing releases
+            for v in self.config["include_versions"]:
+                if re.match(v, version):
+                    break
+            else:
+                self.log("Skipping release whose version doesn't match any "
+                         "include_version pattern: %s" % release_name,
+                         level=INFO)
+                continue
+
+            # We also have to trim out previous releases that aren't in the same
+            # product line, too old, etc.
+            if self.config["stage_product"] != product:
+                self.log("Skipping release that doesn't match product name: %s" % release_name,
+                         level=INFO)
+                continue
+            if MozillaVersion(version) < MozillaVersion(self.config["last_watershed"]):
+                self.log("Skipping release that's behind the last watershed: %s" % release_name,
+                         level=INFO)
+                continue
+            if version == self.config["to_version"]:
+                self.log("Skipping release that is the same as to version: %s" % release_name,
+                         level=INFO)
+                continue
+            if MozillaVersion(version) > MozillaVersion(self.config["to_version"]):
+                self.log("Skipping release that's newer than to version: %s" % release_name,
+                         level=INFO)
+                continue
+
+            if version in self.update_paths:
+                raise Exception("Found duplicate release for version: %s", version)
+
+            # This is a crappy place to get buildids from, but we don't have a better one.
+            # This will start to fail if old info files are deleted.
+            info_file_url = "{}{}/{}_info.txt".format(
+                self.config["previous_archive_prefix"],
+                getCandidatesDir(
+                    self.config["stage_product"],
+                    version,
+                    release_info["build_number"],
+                ),
+                ftp2infoFile(self.config["platform"])
+            )
+            self.log("Retrieving buildid from info file: %s" % info_file_url, level=DEBUG)
+            ret = self._retry_download(info_file_url, "WARNING")
+            buildID = ret.read().split("=")[1].strip()
+
+            shipped_locales_url = urljoin(
+                self.config["hg_server"],
+                "{}/raw-file/{}/{}/locales/shipped-locales".format(
+                    branch,
+                    tag,
+                    self.config["app_name"],
+                ),
+            )
+            ret = self._retry_download(shipped_locales_url, "WARNING")
+            shipped_locales = ret.read().strip()
+
+            app_version_url = urljoin(
+                self.config["hg_server"],
+                "{}/raw-file/{}/{}/config/version.txt".format(
+                    branch,
+                    tag,
+                    self.config["app_name"],
+                ),
+            )
+            app_version = self._retry_download(app_version_url, "WARNING").read().strip()
+
+            self.log("Adding {} to update paths".format(version), level=INFO)
+            self.update_paths[version] = {
+                "appVersion": app_version,
+                "locales": getPlatformLocales(shipped_locales, self.config["platform"]),
+                "buildID": buildID,
+            }
+            for pattern, mar_channel_ids in self.config["mar_channel_id_overrides"].items():
+                if re.match(pattern, version):
+                    self.update_paths[version]["marChannelIds"] = mar_channel_ids
+
+    def gather_info(self):
+        self._get_update_paths()
+        if self.update_paths:
+            self.log("Found update paths:", level=DEBUG)
+            self.log(pprint.pformat(self.update_paths), level=DEBUG)
+        else:
+            self.log("Didn't find any update paths, cannot continue", level=FATAL)
+
+    def create_config(self):
+        from mozrelease.l10n import getPlatformLocales
+        from mozrelease.platforms import ftp2updatePlatforms
+        from mozrelease.update_verify import UpdateVerifyConfig
+        from mozrelease.paths import getCandidatesDir, getReleasesDir, getReleaseInstallerPath
+        from mozrelease.versions import getPrettyVersion
+
+        candidates_dir = getCandidatesDir(
+            self.config["stage_product"], self.config["to_version"],
+            self.config["to_build_number"],
+        )
+        to_ = getReleaseInstallerPath(
+            self.config["product"], self.config["product"].title(),
+            self.config["to_version"], self.config["platform"],
+            locale="%locale%"
+        )
+        to_path = "{}/{}".format(candidates_dir, to_)
+
+        to_display_version = self.config.get("to_display_version")
+        if not to_display_version:
+            to_display_version = getPrettyVersion(self.config["to_version"])
+
+        self.update_verify_config = UpdateVerifyConfig(
+            product=self.config["product"].title(), channel=self.config["channel"],
+            aus_server=self.config["aus_server"], to=to_path,
+            to_build_id=self.config["to_buildid"],
+            to_app_version=self.config["to_app_version"],
+            to_display_version=to_display_version,
+        )
+
+        to_shipped_locales_url = urljoin(
+            self.config["hg_server"],
+            "{}/raw-file/{}/{}/locales/shipped-locales".format(
+                self.config["repo_path"],
+                self.config["to_revision"],
+                self.config["app_name"],
+            ),
+        )
+        to_shipped_locales = self._retry_download(to_shipped_locales_url, "WARNING").read().strip()
+        to_locales = set(getPlatformLocales(to_shipped_locales, self.config["platform"]))
+
+        completes_only_index = 0
+        for fromVersion in reversed(sorted(self.update_paths, key=LooseVersion)):
+            from_ = self.update_paths[fromVersion]
+            locales = sorted(list(set(from_["locales"]).intersection(to_locales)))
+            appVersion = from_["appVersion"]
+            build_id = from_["buildID"]
+            mar_channel_IDs = from_.get('marChannelIds')
+
+            # Use new build targets for Windows, but only on compatible
+            #  versions (42+). See bug 1185456 for additional context.
+            if self.config["platform"] not in ("win32", "win64") or \
+                    LooseVersion(fromVersion) < LooseVersion("42.0"):
+                update_platform = ftp2updatePlatforms(self.config["platform"])[0]
+            else:
+                update_platform = ftp2updatePlatforms(self.config["platform"])[1]
+
+            release_dir = getReleasesDir(
+                self.config["stage_product"], fromVersion
+            )
+            path_ = getReleaseInstallerPath(
+                self.config["product"], self.config["product"].title(),
+                fromVersion, self.config["platform"], locale="%locale%",
+            )
+            from_path = "{}/{}".format(release_dir, path_)
+
+            updater_package = "{}/{}".format(
+                release_dir,
+                getReleaseInstallerPath(
+                    self.config["product"], self.config["product"].title(),
+                    fromVersion, self.config["updater_platform"],
+                    locale="%locale%",
+                )
+            )
+
+            # Exclude locales being full checked
+            quick_check_locales = [l for l in locales
+                                   if l not in self.config["full_check_locales"]]
+            # Get the intersection of from and to full_check_locales
+            this_full_check_locales = [l for l in self.config["full_check_locales"]
+                                       if l in locales]
+
+            if fromVersion in self.config["partial_versions"]:
+                self.info("Generating configs for partial update checks for %s" % fromVersion)
+                self.update_verify_config.addRelease(
+                    release=appVersion, build_id=build_id, locales=locales,
+                    patch_types=["complete", "partial"], from_path=from_path,
+                    ftp_server_from=self.config["previous_archive_prefix"],
+                    ftp_server_to=self.config["archive_prefix"],
+                    mar_channel_IDs=mar_channel_IDs, platform=update_platform,
+                    updater_package=updater_package
+                )
+            else:
+                if this_full_check_locales and is_triangualar(completes_only_index):
+                    self.info("Generating full check configs for %s" % fromVersion)
+                    self.update_verify_config.addRelease(
+                        release=appVersion, build_id=build_id, locales=this_full_check_locales,
+                        from_path=from_path,
+                        ftp_server_from=self.config["previous_archive_prefix"],
+                        ftp_server_to=self.config["archive_prefix"],
+                        mar_channel_IDs=mar_channel_IDs, platform=update_platform,
+                        updater_package=updater_package
+                    )
+                # Quick test for other locales, no download
+                if len(quick_check_locales) > 0:
+                    self.info("Generating quick check configs for %s" % fromVersion)
+                    if not is_triangualar(completes_only_index):
+                        # Assuming we skipped full check locales, using all locales
+                        _locales = locales
+                    else:
+                        # Excluding full check locales from the quick check
+                        _locales = quick_check_locales
+                    self.update_verify_config.addRelease(
+                        release=appVersion, build_id=build_id,
+                        locales=_locales, platform=update_platform
+                    )
+                completes_only_index += 1
+
+    def write_config(self):
+        with open(self.config["output_file"], "w+") as fh:
+            self.update_verify_config.write(fh)
+
+
+if __name__ == "__main__":
+    UpdateVerifyConfigCreator().run_and_exit()