gecko-dev/testing/talos/talos_from_code.py

#! /usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

#
# Script name:   talos_from_code.py
# Purpose:       Read from a talos.json file the different files to download for a talos job
# Author(s):     Zambrano Gasparnian, Armen <armenzg@mozilla.com>
# Target:        Python 2.5
#
import json
import os
import re
import sys
from optparse import OptionParser

import six


def main():
    """
    This script downloads a talos.json file which indicates which files to download
    for a talos job.
    See a talos.json file for a better understand:
    https://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json
    """
    parser = OptionParser()
    parser.add_option(
        "--talos-json-url",
        dest="talos_json_url",
        type="string",
        help="It indicates from where to download the talos.json file.",
    )
    (options, args) = parser.parse_args()

    # 1) check that the url was passed
    if options.talos_json_url is None:
        print("You need to specify --talos-json-url.")
        sys.exit(1)

    # 2) try to download the talos.json file
    try:
        jsonFilename = download_file(options.talos_json_url)
    except Exception as e:
        print("ERROR: We tried to download the talos.json file but something failed.")
        print("ERROR: %s" % str(e))
        sys.exit(1)

    # 3) download the necessary files
    print("INFO: talos.json URL: %s" % options.talos_json_url)
    try:
        key = "talos.zip"
        entity = get_value(jsonFilename, key)
        if passesRestrictions(options.talos_json_url, entity["url"]):
            # the key is at the same time the filename e.g. talos.zip
            print(
                "INFO: Downloading %s as %s"
                % (entity["url"], os.path.join(entity["path"], key))
            )
            download_file(entity["url"], entity["path"], key)
        else:
            print(
                "ERROR: You have tried to download a file "
                + "from: %s " % entity["url"]
                + "which is a location different than http://talos-bundles.pvt.build.mozilla.org/"
            )
            print("ERROR: This is only allowed for the certain branches.")
            sys.exit(1)
    except Exception as e:
        print("ERROR: %s" % str(e))
        sys.exit(1)


def passesRestrictions(talosJsonUrl, fileUrl):
    """
    Only certain branches are exempted from having to host their downloadable files
    in talos-bundles.pvt.build.mozilla.org
    """
    if (
        talosJsonUrl.startswith("http://hg.mozilla.org/try/")
        or talosJsonUrl.startswith("https://hg.mozilla.org/try/")
        or talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/")
        or talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/")
        or talosJsonUrl.startswith("http://hg.mozilla.org/projects/ash/")
        or talosJsonUrl.startswith("https://hg.mozilla.org/projects/ash/")
    ):
        return True
    else:
        p = re.compile("^http://talos-bundles.pvt.build.mozilla.org/")
        m = p.match(fileUrl)
        if m is None:
            return False
        return True


def get_filename_from_url(url):
    """
    This returns the filename of the file we're trying to download
    """
    parsed = six.moves.urllib.parse.urlsplit(url.rstrip("/"))
    if parsed.path != "":
        return parsed.path.rsplit("/", 1)[-1]
    else:
        print(
            "ERROR: We were trying to download a file from %s "
            + "but the URL seems to be incorrect."
        )
        sys.exit(1)


def download_file(url, path="", saveAs=None):
    """
    It downloads a file from URL to the indicated path
    """
    req = six.moves.urllib.request.Request(url)
    f = six.moves.urllib.request.urlopen(req)
    if path != "" and not os.path.isdir(path):
        try:
            os.makedirs(path)
            print("INFO: directory %s created" % path)
        except Exception as e:
            print("ERROR: %s" % str(e))
            sys.exit(1)
    filename = saveAs if saveAs else get_filename_from_url(url)
    local_file = open(os.path.join(path, filename), "wb")
    local_file.write(f.read())
    local_file.close()
    return filename


def get_value(json_filename, key):
    """
    It loads up a JSON file and returns the value for the given string
    """
    f = open(json_filename, "r")
    return json.load(f)[key]


if __name__ == "__main__":
    main()