Bug 1872242 - Change how the last segment of milestone_winversion is generated to improve uniqueness r=firefox-build-system-reviewers,RyanVM,glandium

The previous implementation used days since Jan 1 2000 for the last
16-bit segment. This was not unique enough and caused issues with
Antivirus software if two different channels were built on the same day.

The new approach uses hours since the last milestone bump and uses the
VCS to determine how long ago that was relative to the build time. This
means it will always reset when a new cycle begins, but still be unique
since the digits in the first 3 segments have incremented.

We also now use two of the 16-bits to encode the channel (nightly, beta,
ESR, and release). So two channels built within the same hour will still
be unique.

Using only 14-bits to store the 'hours since version bump', we have
about ~682 days from a version bump before we reach the maximum value we
can store. If a build is done after that point, the segment value will
always be the maximum value for that channel.

Differential Revision: https://phabricator.services.mozilla.com/D200989

config/create_rc.py

@@ -94,10 +94,85 @@ def get_buildid():
     return buildid
 
 
-def days_from_2000_to_buildid(buildid):
-    start = datetime(2000, 1, 1, 0, 0, 0)
+def last_winversion_segment(buildid, app_version_display):
+    """
+    The last segment needs to fit into a 16 bit number. We also need to
+    encode what channel this version is from. We'll do this by using 2 bits
+    to encode the channel, and 14 bits to encode the number of hours since
+    the 'config/milestone.txt' was modified (relative to the build time).
+
+    This gives us about ~682 days of release hours that will yield a unique
+    file version for a specific channel/milestone combination. This should suffice
+    since the main problem we're trying to address is uniqueness in CI for a
+    channel/milestone over about a 1 month period.
+
+    If two builds for the same channel/milestone are done in CI within the same
+    hour there's still a chance for overlap and issues with AV as originally
+    reported in https://bugzilla.mozilla.org/show_bug.cgi?id=1872242
+
+    If a build is done after the ~682 day window of uniqueness, the value for
+    this segment will always be the maximum value for the channel (no overflow).
+    It will also always be the maximum value for the channel if a build is done
+    from a source distribution, because we cannot determine the milestone date
+    change without a VCS.
+
+    If necessary, you can decode the result of this function. You just need to
+    do integer division and divide it by 4. The quotient will be the delta
+    between the milestone bump and the build time, and the remainder will be
+    the channel digit. Refer to the if/else chain near the end of the function
+    for what channels the channel digits map to.
+
+    Example:
+        Encoded: 1544
+
+        1554 / 4 =
+        Quotient: 388
+        Remainder: 2 (ESR)
+    """
+    from mozversioncontrol import MissingVCSTool, get_repository_object
+
+    # Max 16 bit value with 2 most significant bits as 0 (reserved so we can
+    # shift later and make room for the channel digit).
+    MAX_VALUE = 0x3FFF
+
+    try:
+        from pathlib import Path
+
+        topsrcdir = buildconfig.topsrcdir
+        repo = get_repository_object(topsrcdir)
+
+        milestone_time = repo.get_last_modified_time_for_file(
+            Path(topsrcdir) / "config" / "milestone.txt"
+        )
         buildid_time = datetime.strptime(buildid, "%Y%m%d%H%M%S")
-    return (buildid_time - start).days
+
+        time_delta = buildid_time - milestone_time
+        # Convert from seconds to hours
+        # When a build is done more than ~682 days in the future, we can't represent the value.
+        # We'll always set the value to the maximum value instead of overflowing.
+        hours_from_milestone_date = min(
+            int(time_delta.total_seconds() / 3600), MAX_VALUE
+        )
+    except MissingVCSTool:
+        # If we're here we can't use the VCS to determine the time differential, so
+        # we'll just set it to the maximum value instead of doing something weird.
+        hours_from_milestone_date = MAX_VALUE
+        pass
+
+    if buildconfig.substs.get("NIGHTLY_BUILD"):
+        # Nightly
+        channel_digit = 0
+    elif "b" in app_version_display:
+        # Beta
+        channel_digit = 1
+    elif buildconfig.substs.get("MOZ_ESR"):
+        # ESR
+        channel_digit = 2
+    else:
+        # Release
+        channel_digit = 3
+    # left shift to make room to encode the channel digit
+    return str((hours_from_milestone_date << 2) + channel_digit)
 
 
 def digits_only(s):
@@ -127,10 +202,11 @@ def generate_module_rc(binary="", rcinclude=None):
     buildid = get_buildid()
     milestone = buildconfig.substs["GRE_MILESTONE"]
     app_version = buildconfig.substs.get("MOZ_APP_VERSION") or milestone
+    app_version_display = buildconfig.substs.get("MOZ_APP_VERSION_DISPLAY")
     app_winversion = ",".join(split_and_normalize_version(app_version, 4))
     milestone_winversion = ",".join(
         split_and_normalize_version(milestone, 3)
-        + [str(days_from_2000_to_buildid(buildid))]
+        + [last_winversion_segment(buildid, app_version_display)]
     )
     display_name = buildconfig.substs.get("MOZ_APP_DISPLAYNAME", "Mozilla")
 

python/mozversioncontrol/mozversioncontrol/__init__.py

@@ -8,6 +8,7 @@ import os
 import re
 import shutil
 import subprocess
+from datetime import datetime
 from pathlib import Path
 from typing import (
     Iterator,
@@ -345,6 +346,11 @@ class Repository(object):
     def remove_current_commit(self):
         """Remove the currently checked out commit from VCS history."""
 
+    @abc.abstractmethod
+    def get_last_modified_time_for_file(self, path: Path):
+        """Return last modified in VCS time for the specified file."""
+        pass
+
 
 class HgRepository(Repository):
     """An implementation of `Repository` for Mercurial repositories."""
@@ -673,6 +679,20 @@ class HgRepository(Repository):
             self.raise_for_missing_extension("evolve")
             raise
 
+    def get_last_modified_time_for_file(self, path: Path):
+        """Return last modified in VCS time for the specified file."""
+        out = self._run(
+            "log",
+            "--template",
+            "{date|isodatesec}",
+            "--limit",
+            "1",
+            "--follow",
+            str(path),
+        )
+
+        return datetime.strptime(out[:-6].strip(), "%Y-%m-%d %H:%M:%S")
+
 
 class GitRepository(Repository):
     """An implementation of `Repository` for Git repositories."""
@@ -916,6 +936,12 @@ class GitRepository(Repository):
         """Remove the currently checked out commit from VCS history."""
         self._run("reset", "HEAD~")
 
+    def get_last_modified_time_for_file(self, path: Path):
+        """Return last modified in VCS time for the specified file."""
+        out = self._run("log", "-1", "--format=%ad", "--date=iso", path)
+
+        return datetime.strptime(out[:-6].strip(), "%Y-%m-%d %H:%M:%S")
+
 
 class SrcRepository(Repository):
     """An implementation of `Repository` for Git repositories."""
@@ -1037,6 +1063,10 @@ class SrcRepository(Repository):
     def set_config(self, name, value):
         pass
 
+    def get_last_modified_time_for_file(self, path: Path):
+        """Return last modified in VCS time for the specified file."""
+        raise MissingVCSTool
+
 
 def get_repository_object(
     path: Optional[Union[str, Path]], hg="hg", git="git", src="src"

python/mozversioncontrol/test/test_commit.py

@@ -2,6 +2,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from datetime import datetime
+from pathlib import Path
+
 import mozunit
 
 from mozversioncontrol import get_repository_object
@@ -30,14 +33,21 @@ def test_commit(repo):
     repo.execute_next_step()
     assert not vcs.working_directory_clean()
 
+    date_string = "2017-07-14 02:40:00 UTC"
+
     # Commit just bar
     vcs.commit(
         message="Modify bar\n\nbut not baz",
         author="Testing McTesterson <test@example.org>",
-        date="2017-07-14 02:40:00 UTC",
+        date=date_string,
         paths=["bar"],
     )
 
+    original_date = datetime.strptime(date_string[:-4], "%Y-%m-%d %H:%M:%S")
+    date_from_vcs = vcs.get_last_modified_time_for_file(Path("bar"))
+
+    assert original_date == date_from_vcs
+
     # We only committed bar, so foo is still keeping the working dir dirty
     assert not vcs.working_directory_clean()