Bug 1390908 - Make mitmproxy osx binary available on tooltool and update talos tp6 accordingly; r=jmaher

MozReview-Commit-ID: 4YfZMifpq2p

--HG--
extra : rebase_source : 49b186899882e1ffe5865b7c80899101be9bfea2

testing/mozharness/mozharness/mozilla/testing/talos.py

@@ -198,6 +198,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin,
         self.gecko_profile = self.config.get('gecko_profile')
         self.gecko_profile_interval = self.config.get('gecko_profile_interval')
         self.pagesets_name = None
+        self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary
         self.mitmproxy_recording_set = None # zip file found on tooltool that contains all of the mitmproxy recordings
         self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set
         self.mitmdump = None # path to mitdump tool itself, in py3 venv
@@ -457,14 +458,45 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin,
             self.mitmdump = os.path.join(self.py3_path_to_executables(), 'mitmdump')
         else:
             # on macosx we use a prebuilt mitmproxy release binary
-            mitmproxy_bin_url = 'https://github.com/mitmproxy/mitmproxy/releases/download/v2.0.2/mitmproxy-2.0.2-osx.tar.gz'
             mitmproxy_path = os.path.join(self.talos_path, 'talos', 'mitmproxy')
             self.mitmdump = os.path.join(mitmproxy_path, 'mitmdump')
             if not os.path.exists(self.mitmdump):
-                self.download_unpack(mitmproxy_bin_url, mitmproxy_path)
+                # download the mitmproxy release binary; will be overridden by the --no-download
+                if '--no-download' not in self.config['talos_extra_options']:
+                    self.query_mitmproxy_rel_bin('osx')
+                    if self.mitmproxy_rel_bin is None:
+                        self.fatal("Aborting: mitmproxy_release_bin_osx not found in talos.json")
+                    self.download_mitmproxy_binary('osx')
+                else:
+                    self.info("Not downloading mitmproxy rel binary because no-download was specified")
             self.info('The mitmdump macosx binary is found at: %s' % self.mitmdump)
         self.run_command([self.mitmdump, '--version'], env=self.query_env())
 
+    def query_mitmproxy_rel_bin(self, platform):
+        """Mitmproxy requires external playback archives to be downloaded and extracted"""
+        if self.mitmproxy_rel_bin:
+            return self.mitmproxy_rel_bin
+        if self.query_talos_json_config() and self.suite is not None:
+            config_key = "mitmproxy_release_bin_" + platform
+            self.mitmproxy_rel_bin = self.talos_json_config['suites'][self.suite].get(config_key, False)
+            return self.mitmproxy_rel_bin
+
+    def download_mitmproxy_binary(self, platform):
+        """Download the mitmproxy release binary from tooltool"""
+        self.info("Downloading the mitmproxy release binary using tooltool")
+        dest = os.path.join(self.talos_path, 'talos', 'mitmproxy')
+        _manifest = "mitmproxy-rel-bin-%s.manifest" % platform
+        manifest_file = os.path.join(self.talos_path, 'talos', 'mitmproxy', _manifest)
+        self.tooltool_fetch(
+            manifest_file,
+            output_dir=dest,
+            cache=self.config.get('tooltool_cache')
+        )
+        archive = os.path.join(dest, self.mitmproxy_rel_bin)
+        tar = self.query_exe('tar')
+        unzip_cmd = [tar, '-xvzf', archive, '-C', dest]
+        self.run_command(unzip_cmd, halt_on_failure=True)
+
     def query_mitmproxy_recording_set(self):
         """Mitmproxy requires external playback archives to be downloaded and extracted"""
         if self.mitmproxy_recording_set:

testing/talos/talos.json

@@ -111,6 +111,7 @@
         },
         "tp6-e10s": {
             "tests": ["tp6_google", "tp6_youtube", "tp6_amazon", "tp6_facebook"],
+            "mitmproxy_release_bin_osx": "mitmproxy-2.0.2-osx.tar.gz",
             "mitmproxy_recording_set": "mitmproxy-recording-set-win10.zip",
             "talos_options": [
                 "--mitmproxy",
@@ -120,6 +121,7 @@
         },
         "tp6-stylo-disabled-e10s": {
             "tests": ["tp6_google", "tp6_youtube", "tp6_amazon", "tp6_facebook"],
+            "mitmproxy_release_bin_osx": "mitmproxy-2.0.2-osx.tar.gz",
             "mitmproxy_recording_set": "mitmproxy-recording-set-win10.zip",
             "talos_options": [
                 "--disable-stylo",
@@ -130,6 +132,7 @@
         },
         "tp6-stylo-threads-e10s": {
             "tests": ["tp6_google", "tp6_youtube", "tp6_amazon", "tp6_facebook"],
+            "mitmproxy_release_bin_osx": "mitmproxy-2.0.2-osx.tar.gz",
             "mitmproxy_recording_set": "mitmproxy-recording-set-win10.zip",
             "talos_options": [
                 "--stylo-threads=1",

testing/talos/talos/mitmproxy/mitmproxy-rel-bin-osx.manifest

@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "mitmproxy-2.0.2-osx.tar.gz",
+        "size": 32324573,
+        "digest": "06423c76e7e99fd9705eae3dc6e2423b1ffb8c42caa98fd010d59dc6ed1f0827376e238c48108106da558444b826e085a58aeb30cf9c79e9d0122a2cb17ae8e6",
+        "algorithm": "sha512",
+        "unpack": false
+    }
+]