Bug 1649176 - add batch attribute to add --test-date r=sparky

Right now setup_perftest_test_date adds --test-date yesterday to all perftest
runs. we want that only for the ones doing batches

Differential Revision: https://phabricator.services.mozilla.com/D81562

python/mozperftest/mozperftest/argparser.py

@@ -18,6 +18,7 @@ except ImportError:
 from mozperftest.system import get_layers as system_layers  # noqa
 from mozperftest.test import get_layers as test_layers  # noqa
 from mozperftest.metrics import get_layers as metrics_layers  # noqa
+from mozperftest.utils import convert_day  # noqa
 
 FLAVORS = ["desktop-browser", "mobile-browser", "doc"]
 
@@ -72,10 +73,10 @@ class Options:
             "help": "Running the test on try",
         },
         "--test-date": {
-            "type": str,
-            "default": "yesterday",
+            "type": convert_day,
+            "default": "today",
             "help": "Used in multi-commit testing, it specifies the day to get test builds from. "
-            "Must follow the format `YYYY.MM.DD`.",
+            "Must follow the format `YYYY.MM.DD` or be `today` or `yesterday`.",
         },
     }
 

python/mozperftest/mozperftest/mach_commands.py

@@ -130,6 +130,8 @@ class PerftestTests(MachCommandBase):
         except ImportError:
             pydeps = Path(self.topsrcdir, "third_party", "python")
             vendors = ["coverage"]
+            if not ON_TRY:
+                vendors.append("attrs")
             if skip_linters:
                 pypis = []
             else:

python/mozperftest/mozperftest/tests/test_argparser.py

@@ -3,6 +3,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import mozunit
+import pytest
+from datetime import date
 
 from mozperftest.argparser import PerftestArgumentParser, Options
 
@@ -21,5 +23,19 @@ def test_options():
     )
 
 
+def test_bad_test_date():
+    parser = PerftestArgumentParser()
+    args = ["test_one.js", "--test-date", "bleh"]
+    with pytest.raises(SystemExit):
+        parser.parse_args(args)
+
+
+def test_test_date_today():
+    parser = PerftestArgumentParser()
+    args = ["test_one.js", "--test-date", "today"]
+    res = parser.parse_args(args)
+    assert res.test_date == date.today().strftime("%Y.%m.%d")
+
+
 if __name__ == "__main__":
     mozunit.main()

python/mozperftest/mozperftest/tests/test_utils.py

@@ -8,6 +8,7 @@ from unittest import mock
 import pytest
 from pathlib import Path
 import shutil
+from datetime import date, timedelta
 
 from mozperftest.utils import (
     host_platform,
@@ -16,6 +17,7 @@ from mozperftest.utils import (
     install_package,
     build_test_list,
     get_multi_tasks_url,
+    convert_day,
 )
 from mozperftest.tests.support import temp_file, requests_content, EXAMPLE_TESTS_DIR
 
@@ -88,6 +90,17 @@ def test_build_test_list():
         shutil.rmtree(tmp_dir)
 
 
+def test_convert_day():
+    day = "2020.06.08"
+    assert convert_day(day) == day
+    with pytest.raises(ValueError):
+        convert_day("2020-06-08")
+    today = date.today()
+    assert convert_day("today"), today.strftime("%Y.%m.%d")
+    yesterday = today - timedelta(days=1)
+    assert convert_day("yesterday") == yesterday.strftime("%Y.%m.%d")
+
+
 def test_multibuild_url():
     route = "FakeBuildRoute"
     day = "2020.06.08"

python/mozperftest/mozperftest/utils.py

@@ -3,7 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import logging
 import contextlib
-from datetime import date, timedelta
+from datetime import datetime, date, timedelta
 import sys
 import os
 import random
@@ -195,15 +195,25 @@ def temporary_env(**env):
                 os.environ[key] = value
 
 
+def convert_day(day):
+    if day in ("yesterday", "today"):
+        curr = date.today()
+        if day == "yesterday":
+            curr = curr - timedelta(1)
+        day = curr.strftime("%Y.%m.%d")
+    else:
+        # verify that the user provided string is in the expected format
+        # if it can't parse it, it'll raise a value error
+        datetime.strptime(day, "%Y.%m.%d")
+
+    return day
+
+
 def get_multi_tasks_url(route, day="yesterday"):
     """Builds a URL to obtain all the tasks of a given build route for a single day.
 
     If previous is true, then we get builds from the previous day,
     otherwise, we look at the current day.
     """
-    if day in ("yesterday", "today"):
-        curr = date.today()
-        if day == "yesterday":
-            curr = curr - timedelta(1)
-        day = curr.strftime("%Y.%m.%d")
+    day = convert_day(day)
     return f"""{MULTI_TASK_ROOT}{route}.{day}.revision"""

taskcluster/ci/perftest/kind.yml

@@ -66,6 +66,7 @@ jobs:
         platform: android-hw-g5-7-0-arm7-api-16/opt
         attributes:
             cron: true
+            batch: true
         run:
             command: >-
                 mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
@@ -96,6 +97,7 @@ jobs:
         platform: android-hw-p2-8-0-android-aarch64/opt
         attributes:
             cron: true
+            batch: true
         run:
             command: >-
                 mkdir -p $MOZ_FETCHES_DIR/../artifacts &&

taskcluster/docs/attributes.rst

@@ -296,6 +296,11 @@ artifact_map
 For beetmover jobs, this indicates which yaml file should be used to
 generate the upstream artifacts and payload instructions to the task.
 
+batch
+=====
+Used by `perftest` to indicates that a task can be run as a batch.
+
+
 enable-full-crashsymbols
 ========================
 In automation, full crashsymbol package generation is normally disabled.  For

taskcluster/taskgraph/transforms/perftest.py

@@ -13,6 +13,7 @@ import six
 
 from taskgraph.transforms.base import TransformSequence
 
+
 transforms = TransformSequence()
 
 
@@ -29,7 +30,8 @@ def pass_perftest_options(config, jobs):
 @transforms.add
 def setup_perftest_test_date(config, jobs):
     for job in jobs:
-        if "--test-date" not in job["run"]["command"]:
-            job["run"]["command"] += " --test-date %s" % \
-                (date.today() - timedelta(1)).strftime("%Y.%m.%d")
+        if (job.get("attributes", {}).get("batch", False) and
+            "--test-date" not in job["run"]["command"]):
+            yesterday = (date.today() - timedelta(1)).strftime("%Y.%m.%d")
+            job["run"]["command"] += " --test-date %s" % yesterday
         yield job