diff --git a/config/create_rc.py b/config/create_rc.py index d75959d4e02b..b429142e157d 100644 --- a/config/create_rc.py +++ b/config/create_rc.py @@ -94,10 +94,85 @@ def get_buildid(): return buildid -def days_from_2000_to_buildid(buildid): - start = datetime(2000, 1, 1, 0, 0, 0) - buildid_time = datetime.strptime(buildid, "%Y%m%d%H%M%S") - return (buildid_time - start).days +def last_winversion_segment(buildid, app_version_display): + """ + The last segment needs to fit into a 16 bit number. We also need to + encode what channel this version is from. We'll do this by using 2 bits + to encode the channel, and 14 bits to encode the number of hours since + the 'config/milestone.txt' was modified (relative to the build time). + + This gives us about ~682 days of release hours that will yield a unique + file version for a specific channel/milestone combination. This should suffice + since the main problem we're trying to address is uniqueness in CI for a + channel/milestone over about a 1 month period. + + If two builds for the same channel/milestone are done in CI within the same + hour there's still a chance for overlap and issues with AV as originally + reported in https://bugzilla.mozilla.org/show_bug.cgi?id=1872242 + + If a build is done after the ~682 day window of uniqueness, the value for + this segment will always be the maximum value for the channel (no overflow). + It will also always be the maximum value for the channel if a build is done + from a source distribution, because we cannot determine the milestone date + change without a VCS. + + If necessary, you can decode the result of this function. You just need to + do integer division and divide it by 4. The quotient will be the delta + between the milestone bump and the build time, and the remainder will be + the channel digit. Refer to the if/else chain near the end of the function + for what channels the channel digits map to. + + Example: + Encoded: 1544 + + 1554 / 4 = + Quotient: 388 + Remainder: 2 (ESR) + """ + from mozversioncontrol import MissingVCSTool, get_repository_object + + # Max 16 bit value with 2 most significant bits as 0 (reserved so we can + # shift later and make room for the channel digit). + MAX_VALUE = 0x3FFF + + try: + from pathlib import Path + + topsrcdir = buildconfig.topsrcdir + repo = get_repository_object(topsrcdir) + + milestone_time = repo.get_last_modified_time_for_file( + Path(topsrcdir) / "config" / "milestone.txt" + ) + buildid_time = datetime.strptime(buildid, "%Y%m%d%H%M%S") + + time_delta = buildid_time - milestone_time + # Convert from seconds to hours + # When a build is done more than ~682 days in the future, we can't represent the value. + # We'll always set the value to the maximum value instead of overflowing. + hours_from_milestone_date = min( + int(time_delta.total_seconds() / 3600), MAX_VALUE + ) + except MissingVCSTool: + # If we're here we can't use the VCS to determine the time differential, so + # we'll just set it to the maximum value instead of doing something weird. + hours_from_milestone_date = MAX_VALUE + pass + + if buildconfig.substs.get("NIGHTLY_BUILD"): + # Nightly + channel_digit = 0 + elif "b" in app_version_display: + # Beta + channel_digit = 1 + elif buildconfig.substs.get("MOZ_ESR"): + # ESR + channel_digit = 2 + else: + # Release + channel_digit = 3 + # left shift to make room to encode the channel digit + return str((hours_from_milestone_date << 2) + channel_digit) def digits_only(s): @@ -127,10 +202,11 @@ def generate_module_rc(binary="", rcinclude=None): buildid = get_buildid() milestone = buildconfig.substs["GRE_MILESTONE"] app_version = buildconfig.substs.get("MOZ_APP_VERSION") or milestone + app_version_display = buildconfig.substs.get("MOZ_APP_VERSION_DISPLAY") app_winversion = ",".join(split_and_normalize_version(app_version, 4)) milestone_winversion = ",".join( split_and_normalize_version(milestone, 3) - + [str(days_from_2000_to_buildid(buildid))] + + [last_winversion_segment(buildid, app_version_display)] ) display_name = buildconfig.substs.get("MOZ_APP_DISPLAYNAME", "Mozilla") diff --git a/python/mozversioncontrol/mozversioncontrol/__init__.py b/python/mozversioncontrol/mozversioncontrol/__init__.py index 32e9705e3460..d996656d17d1 100644 --- a/python/mozversioncontrol/mozversioncontrol/__init__.py +++ b/python/mozversioncontrol/mozversioncontrol/__init__.py @@ -8,6 +8,7 @@ import os import re import shutil import subprocess +from datetime import datetime from pathlib import Path from typing import ( Iterator, @@ -345,6 +346,11 @@ class Repository(object): def remove_current_commit(self): """Remove the currently checked out commit from VCS history.""" + @abc.abstractmethod + def get_last_modified_time_for_file(self, path: Path): + """Return last modified in VCS time for the specified file.""" + pass + class HgRepository(Repository): """An implementation of `Repository` for Mercurial repositories.""" @@ -673,6 +679,20 @@ class HgRepository(Repository): self.raise_for_missing_extension("evolve") raise + def get_last_modified_time_for_file(self, path: Path): + """Return last modified in VCS time for the specified file.""" + out = self._run( + "log", + "--template", + "{date|isodatesec}", + "--limit", + "1", + "--follow", + str(path), + ) + + return datetime.strptime(out[:-6].strip(), "%Y-%m-%d %H:%M:%S") + class GitRepository(Repository): """An implementation of `Repository` for Git repositories.""" @@ -916,6 +936,12 @@ class GitRepository(Repository): """Remove the currently checked out commit from VCS history.""" self._run("reset", "HEAD~") + def get_last_modified_time_for_file(self, path: Path): + """Return last modified in VCS time for the specified file.""" + out = self._run("log", "-1", "--format=%ad", "--date=iso", path) + + return datetime.strptime(out[:-6].strip(), "%Y-%m-%d %H:%M:%S") + class SrcRepository(Repository): """An implementation of `Repository` for Git repositories.""" @@ -1037,6 +1063,10 @@ class SrcRepository(Repository): def set_config(self, name, value): pass + def get_last_modified_time_for_file(self, path: Path): + """Return last modified in VCS time for the specified file.""" + raise MissingVCSTool + def get_repository_object( path: Optional[Union[str, Path]], hg="hg", git="git", src="src" diff --git a/python/mozversioncontrol/test/test_commit.py b/python/mozversioncontrol/test/test_commit.py index b795c0ea6e64..2244a652a731 100644 --- a/python/mozversioncontrol/test/test_commit.py +++ b/python/mozversioncontrol/test/test_commit.py @@ -2,6 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from datetime import datetime +from pathlib import Path + import mozunit from mozversioncontrol import get_repository_object @@ -30,14 +33,21 @@ def test_commit(repo): repo.execute_next_step() assert not vcs.working_directory_clean() + date_string = "2017-07-14 02:40:00 UTC" + # Commit just bar vcs.commit( message="Modify bar\n\nbut not baz", author="Testing McTesterson ", - date="2017-07-14 02:40:00 UTC", + date=date_string, paths=["bar"], ) + original_date = datetime.strptime(date_string[:-4], "%Y-%m-%d %H:%M:%S") + date_from_vcs = vcs.get_last_modified_time_for_file(Path("bar")) + + assert original_date == date_from_vcs + # We only committed bar, so foo is still keeping the working dir dirty assert not vcs.working_directory_clean()