From 6427d63dfee421d64306ed8e1f365296b4c5a36b Mon Sep 17 00:00:00 2001
From: Joel Maher <jmaher@mozilla.com>
Date: Mon, 20 Nov 2023 17:51:48 +0000
Subject: [PATCH] Bug 1865560 - fix test-info-all to reduce queries and include
 total_runs. r=gbrown

Differential Revision: https://phabricator.services.mozilla.com/D194026
---
 taskcluster/ci/source-test/file-metadata.yml |  2 +-
 testing/mach_commands.py                     |  3 +++
 testing/testinfo.py                          | 19 +++++++++++++++----
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/taskcluster/ci/source-test/file-metadata.yml b/taskcluster/ci/source-test/file-metadata.yml
index 49c89d2561b7..b1fffaf64fe4 100644
--- a/taskcluster/ci/source-test/file-metadata.yml
+++ b/taskcluster/ci/source-test/file-metadata.yml
@@ -122,5 +122,5 @@ test-info-all:
         command: >-
             source taskcluster/scripts/misc/source-test-common.sh &&
             ./mach test-info testrun-report --output-file /builds/worker/artifacts/test-run-info.json &&
-            ./mach test-info report --show-tests --show-summary --show-testruns --verbose --output-file /builds/worker/artifacts/test-info-all-tests.json &&
+            ./mach test-info report --show-tests --show-summary --show-testruns --verbose --output-file /builds/worker/artifacts/test-info-all-tests.json --runcounts-input-file /builds/worker/artifacts/test-run-info.json &&
             ./mach test-info report --show-annotations --output-file /builds/worker/artifacts/test-info-manifest-conditions.json
diff --git a/testing/mach_commands.py b/testing/mach_commands.py
index f8b14b190995..6a2474fb1cc8 100644
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -903,6 +903,7 @@ def test_info_tests(
     help="Do not categorize by bugzilla component.",
 )
 @CommandArgument("--output-file", help="Path to report file.")
+@CommandArgument("--runcounts-input-file", help="Optional path to report file.")
 @CommandArgument("--verbose", action="store_true", help="Enable debug logging.")
 @CommandArgument(
     "--start",
@@ -930,6 +931,7 @@ def test_report(
     start,
     end,
     show_testruns,
+    runcounts_input_file,
 ):
     import testinfo
     from mozbuild import build_commands
@@ -957,6 +959,7 @@ def test_report(
         start,
         end,
         show_testruns,
+        runcounts_input_file,
     )
 
 
diff --git a/testing/testinfo.py b/testing/testinfo.py
index e3618effbbc0..bdb845093f4d 100644
--- a/testing/testinfo.py
+++ b/testing/testinfo.py
@@ -234,7 +234,9 @@ class TestInfoReport(TestInfo):
         # if we fail to get valid json (i.e. end point has malformed data), return {}
         retVal = {}
         try:
+            self.log_verbose("getting url: %s" % target_url)
             r = requests.get(target_url, headers={"User-agent": "mach-test-info/1.0"})
+            self.log_verbose("got status: %s" % r.status_code)
             r.raise_for_status()
             retVal = r.json()
         except json.decoder.JSONDecodeError:
@@ -372,9 +374,17 @@ class TestInfoReport(TestInfo):
             return name_part.split()[-1]  # get just the test name, not extra words
         return None
 
-    def get_runcount_data(self, start, end):
+    def get_runcount_data(self, runcounts_input_file, start, end):
         # TODO: use start/end properly
-        runcounts = self.get_runcounts(days=MAX_DAYS)
+        if runcounts_input_file:
+            try:
+                with open(runcounts_input_file, "r") as f:
+                    runcounts = json.load(f)
+            except:
+                print("Unable to load runcounts from path: %s" % runcounts_input_file)
+                raise
+        else:
+            runcounts = self.get_runcounts(days=MAX_DAYS)
         runcounts = self.squash_runcounts(runcounts, days=MAX_DAYS)
         return runcounts
 
@@ -557,6 +567,7 @@ class TestInfoReport(TestInfo):
         start,
         end,
         show_testruns,
+        runcounts_input_file,
     ):
         def matches_filters(test):
             """
@@ -604,7 +615,7 @@ class TestInfoReport(TestInfo):
             "https://hg.mozilla.org/mozilla-central",
             "https://hg.mozilla.org/try",
         ]:
-            runcount = self.get_runcount_data(start, end)
+            runcount = self.get_runcount_data(runcounts_input_file, start, end)
 
         print("Finding tests...")
         here = os.path.abspath(os.path.dirname(__file__))
@@ -783,7 +794,7 @@ class TestInfoReport(TestInfo):
                                 total_runs = 0
                                 for m in test_info["manifest"]:
                                     if m in runcount.keys():
-                                        for x in runcount.get("m", []):
+                                        for x in runcount.get(m, []):
                                             if not x:
                                                 break
                                             total_runs += x[3]