Bug 1758584: Add in-proc venv activation paths to the end of sys.path r=ahal

So far, we've been using `virtualenv`'s `activate_this.py` script.
However, unlike earlier expectations, it adds its `sys.path` additions
to the //front//, not the back! This breaks our prioritization
requirements, such as:
* When using any package from the system environment, import *all
  possible* packages from the system to avoid compatibility issues.
* Use vendored packages instead of virtualenv-installed packages
  wherever possible, because it more-closely matches developer
  expectations ("why is this package vendored if it's not used?")

Define an `activate_virtualenv()` function that replicates the logic
of `activate_this.py` [1], except for three differences:
* Don't modify `sys.real_prefix`, since it's a non-standard property of
  `sys`.
* Only add seen-with-`venv`-module paths to the `sys.path` (`$prefix`,
  `$prefix/.../$site_packages_dir`) - don't do the paths in-between.
* And, of course, append instead of prepend `sys.path` entries.

As an aside, this is one of the few remaining blockers from allowing
us to fully embrace `venv` instead of `virtualenv` - the last piece is
waiting on the fix for bug 1697833 to propagate.

[1]
https://github.com/pypa/virtualenv/blob/20.7.2/src/virtualenv/activation/python/activate_this.py

Differential Revision: https://phabricator.services.mozilla.com/D140579

python/mach/mach/site.py

@@ -14,6 +14,7 @@ import json
 import os
 import platform
 import shutil
+import site
 import subprocess
 import sys
 from collections import OrderedDict
@@ -364,8 +365,7 @@ class MachSiteManager:
                     # automatically adds the virtualenv's "site-packages" to our scope, in
                     # addition to our first-party/vendored modules since they're specified
                     # in the "mach.pth" file.
-                    activate_path = self._virtualenv().activate_path
-                    exec(open(activate_path).read(), dict(__file__=activate_path))
+                    activate_virtualenv(self._virtualenv())
 
     def _build(self):
         if self._site_packages_source != SitePackagesSource.VENV:
@@ -552,8 +552,7 @@ class CommandSiteManager:
         self.ensure()
 
         with self._metadata.update_current_site(self._virtualenv.python_path):
-            activate_path = self._virtualenv.activate_path
-            exec(open(activate_path).read(), dict(__file__=activate_path))
+            activate_virtualenv(self._virtualenv)
 
     def install_pip_package(self, package):
         """Install a package via pip.
@@ -746,7 +745,6 @@ class PythonVirtualenv:
         else:
             self.bin_path = os.path.join(prefix, "bin")
             self.python_path = os.path.join(self.bin_path, "python")
-        self.activate_path = os.path.join(self.bin_path, "activate_this.py")
         self.prefix = prefix
 
     @functools.lru_cache(maxsize=None)
@@ -1133,8 +1131,6 @@ def _create_venv_with_pthfile(
         ]
     )
 
-    os.utime(target_venv.activate_path, None)
-
     site_packages_dir = target_venv.site_packages_dir()
     pthfile_contents = "\n".join(pthfile_lines)
     with open(os.path.join(site_packages_dir, PTH_FILENAME), "w") as f:
@@ -1145,7 +1141,6 @@ def _create_venv_with_pthfile(
             target_venv.pip_install([str(requirement.requirement)])
         target_venv.install_optional_packages(requirements.pypi_optional_requirements)
 
-    os.utime(target_venv.activate_path, None)
     metadata.write(is_finalized=True)
 
 
@@ -1156,9 +1151,7 @@ def _is_venv_up_to_date(
     requirements,
     expected_metadata,
 ):
-    if not os.path.exists(target_venv.prefix) or not os.path.exists(
-        target_venv.activate_path
-    ):
+    if not os.path.exists(target_venv.prefix):
         return False
 
     virtualenv_package = os.path.join(
@@ -1176,9 +1169,11 @@ def _is_venv_up_to_date(
     # * This file
     # * The `virtualenv` package
     # * Any of our requirements manifest files
-    activate_mtime = os.path.getmtime(target_venv.activate_path)
+    metadata_mtime = os.path.getmtime(
+        os.path.join(target_venv.prefix, METADATA_FILENAME)
+    )
     dep_mtime = max(os.path.getmtime(p) for p in deps)
-    if dep_mtime > activate_mtime:
+    if dep_mtime > metadata_mtime:
         return False
 
     try:
@@ -1205,5 +1200,17 @@ def _is_venv_up_to_date(
     return True
 
 
+def activate_virtualenv(virtualenv: PythonVirtualenv):
+    os.environ["PATH"] = os.pathsep.join(
+        [virtualenv.bin_path] + os.environ.get("PATH", "").split(os.pathsep)
+    )
+    os.environ["VIRTUAL_ENV"] = virtualenv.prefix
+
+    for path in (virtualenv.prefix, virtualenv.site_packages_dir()):
+        site.addsitedir(os.path.realpath(path))
+
+    sys.prefix = virtualenv.prefix
+
+
 def _mach_virtualenv_root(checkout_scoped_state_dir):
     return os.path.join(checkout_scoped_state_dir, "_virtualenvs", "mach")

python/mach/mach/test/test_site_activation.py

@@ -11,7 +11,7 @@ import tempfile
 
 from buildconfig import topsrcdir
 import mozunit
-from mach.site import MozSiteMetadata, PythonVirtualenv
+from mach.site import MozSiteMetadata, PythonVirtualenv, activate_virtualenv
 import pkg_resources
 
 
@@ -41,11 +41,10 @@ def test_new_package_appears_in_pkg_resources():
 
         venv = PythonVirtualenv(venv_dir)
         venv.pip_install(["carrot==0.10.7"])
-        activate_path = venv.activate_path
 
         metadata = MozSiteMetadata(None, None, None, None, None, venv.prefix)
         with metadata.update_current_site(venv.python_path):
-            exec(open(activate_path).read(), dict(__file__=activate_path))
+            activate_virtualenv(venv)
 
         assert pkg_resources.get_distribution("carrot").version == "0.10.7"