Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-23 12:51:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1730712: Update "Moz site" terminology r=ahal

Browse Source 
The existing terminology had two issues:
* `VirtualenvManager` wasn't always associated with an on-disk
  `virtualenv`: for example, when running in automation, Mach
  "activates" a `VirtualenvManager`, updating its import scope,
  but without ever creating an on-disk `virtualenv`.
* An upcoming patch splits the `VirtualenvManager` class, pulling
  "on-disk virtualenv-handling functions" from the project-wide
  interface for managing Python's import scope.

After some good discussion with Ahal, I think we've struck
the terminology that handles this distinction well: we'll call
the "import scope"-handling part the "site", and we'll continue
to call on-disk virtualenvs (and their representative classes)
as, well, virtualenvs.

Differential Revision: https://phabricator.services.mozilla.com/D130391
This commit is contained in:
Mitchell Hentges 2021-11-17 20:54:23 +00:00
parent b8ff587b0d
commit 725792bd05
14 changed files with 78 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build/docs/python.rst
View File

@ -73,7 +73,7 @@ to just keep them separate so there is no potential for conflicts.
Very early in the build process, a virtualenv is created inside the
:term:`object directory`. The virtualenv is configured such that it can
find all the Python packages in the source tree. The code for this lives
in :py:mod:`mach.virtualenv`.
in :py:mod:`mach.site`.
Deficiencies
------------

26
build/mach_initialize.py
View File

@ -174,7 +174,7 @@ def _scrub_system_site_packages():
def _activate_python_environment(topsrcdir, state_dir):
# We need the "mach" module to access the logic to parse virtualenv
# We need the "mach" module to access the logic to parse site
# requirements. Since that depends on "packaging" (and, transitively,
# "pyparsing"), we add those to the path too.
sys.path[0:0] = [
@ -186,28 +186,26 @@ def _activate_python_environment(topsrcdir, state_dir):
)
]
from mach.virtualenv import (
MozVirtualenvMetadata,
MozVirtualenvMetadataOutOfDateError,
VirtualenvManager,
from mach.site import (
MozSiteMetadata,
MozSiteMetadataOutOfDateError,
MozSiteManager,
)
try:
mach_virtualenv = VirtualenvManager(
mach_site = MozSiteManager(
topsrcdir,
os.path.join(state_dir, "_virtualenvs"),
"mach",
)
active_metadata = MozVirtualenvMetadata.from_runtime()
is_mach_virtualenv = (
active_metadata and active_metadata.virtualenv_name == "mach"
)
except MozVirtualenvMetadataOutOfDateError as e:
active_site_metadata = MozSiteMetadata.from_runtime()
is_mach_site = active_site_metadata and active_site_metadata.site_name == "mach"
except MozSiteMetadataOutOfDateError as e:
print(e)
print('This should be resolved by running "./mach create-mach-environment".')
sys.exit(1)
requirements = mach_virtualenv.requirements()
requirements = mach_site.requirements()
if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"):
env_var = (
"MOZ_AUTOMATION"
@ -261,14 +259,14 @@ def _activate_python_environment(topsrcdir, state_dir):
_scrub_system_site_packages()
sys.path[0:0] = requirements.pths_as_absolute(topsrcdir)
elif is_mach_virtualenv:
elif is_mach_site:
# We're running in the Mach virtualenv - check that it's up-to-date.
# Note that the "pip package check" exists to ensure that a virtualenv isn't
# corrupted by ad-hoc pip installs. Since the Mach virtualenv is unlikely
# to be affected by such installs, and since it takes ~400ms to get the list
# of installed pip packages (a *lot* of time to wait during Mach init), we
# skip verifying that our pip packages exist.
if not mach_virtualenv.up_to_date(skip_pip_package_check=True):
if not mach_site.up_to_date(skip_pip_package_check=True):
print(
'The "mach" virtualenv is not up-to-date, please run '
'"./mach create-mach-environment"'

2
build/moz.configure/init.configure
View File

@ -218,7 +218,7 @@ shell = help_shell | shell
@dependable
@checking("for Python 3", callback=lambda x: "%s (%s)" % (x.path, x.str_version))
@imports("sys")
@imports(_from="mach.virtualenv", _import="VirtualenvHelper")
@imports(_from="mach.site", _import="VirtualenvHelper")
def virtualenv_python3():
return namespace(
# sys.executable is currently not updated for in-process activations. However,

10
configure.py
View File

@ -22,7 +22,7 @@ sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
from mach.virtualenv import VirtualenvManager
from mach.site import MozSiteManager
from mozbuild.configure import (
ConfigureSandbox,
TRACE,
@ -232,15 +232,15 @@ def _activate_build_virtualenv():
topobjdir = os.path.realpath(".")
topsrcdir = os.path.realpath(os.path.dirname(__file__))
build_venv = VirtualenvManager(
build_site = MozSiteManager(
topsrcdir,
os.path.join(topobjdir, "_virtualenvs"),
"build",
)
if not build_venv.up_to_date():
if not build_site.up_to_date():
print("Creating Python 3 virtualenv")
build_venv.build()
build_venv.activate()
build_site.build()
build_site.activate()
if __name__ == "__main__":

2
moz.configure
View File

@ -1138,7 +1138,7 @@ def config_status_deps(build_env, build_project):
os.path.join(topsrcdir, "build", "build_virtualenv_packages.txt"),
os.path.join(topsrcdir, "build", "common_virtualenv_packages.txt"),
os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
os.path.join(topsrcdir, "python", "mach", "mach", "virtualenv.py"),
os.path.join(topsrcdir, "python", "mach", "mach", "site.py"),
os.path.join(topsrcdir, "aclocal.m4"),
os.path.join(topsrcdir, "old-configure.in"),
os.path.join(topsrcdir, "js", "src", "aclocal.m4"),

26
python/mach/mach/requirements.py
View File

@ -53,18 +53,18 @@ class PypiOptionalSpecifier(PypiSpecifier):
class MachEnvRequirements:
"""Requirements associated with a "virtualenv_packages.txt" definition
Represents the dependencies of a virtualenv. The source files consist
Represents the dependencies of a site. The source files consist
of colon-delimited fields. The first field
specifies the action. The remaining fields are arguments to that
action. The following actions are supported:
pth -- Adds the path given as argument to "mach.pth" under
the virtualenv site packages directory.
the virtualenv's site packages directory.
pypi -- Fetch the package, plus dependencies, from PyPI.
pypi-optional -- Attempt to install the package and dependencies from PyPI.
Continue using the virtualenv, even if the package could not be installed.
Continue using the site, even if the package could not be installed.
packages.txt -- Denotes that the specified path is a child manifest. It
will be read and processed as if its contents were concatenated
@ -124,7 +124,7 @@ class MachEnvRequirements:
cls,
topsrcdir,
is_thunderbird,
is_mach_or_build_virtualenv,
is_mach_or_build_site,
requirements_definition,
):
requirements = cls()
@ -133,7 +133,7 @@ class MachEnvRequirements:
requirements_definition,
topsrcdir,
is_thunderbird,
is_mach_or_build_virtualenv,
is_mach_or_build_site,
)
return requirements
@ -143,7 +143,7 @@ def _parse_mach_env_requirements(
root_requirements_path,
topsrcdir,
is_thunderbird,
is_mach_or_build_virtualenv,
is_mach_or_build_site,
):
topsrcdir = Path(topsrcdir)
@ -188,9 +188,7 @@ def _parse_mach_env_requirements(
raise Exception(THUNDERBIRD_PYPI_ERROR)
requirements_output.pypi_requirements.append(
PypiSpecifier(
_parse_package_specifier(params, is_mach_or_build_virtualenv)
)
PypiSpecifier(_parse_package_specifier(params, is_mach_or_build_site))
)
elif action == "pypi-optional":
if is_thunderbird_packages_txt:
@ -206,9 +204,7 @@ def _parse_mach_env_requirements(
requirements_output.pypi_optional_requirements.append(
PypiOptionalSpecifier(
repercussion,
_parse_package_specifier(
raw_requirement, is_mach_or_build_virtualenv
),
_parse_package_specifier(raw_requirement, is_mach_or_build_site),
)
)
elif action == "thunderbird-packages.txt":
@ -237,14 +233,14 @@ def _parse_mach_env_requirements(
_parse_requirements_definition_file(root_requirements_path, False)
def _parse_package_specifier(raw_requirement, is_mach_or_build_virtualenv):
def _parse_package_specifier(raw_requirement, is_mach_or_build_site):
requirement = Requirement(raw_requirement)
if not is_mach_or_build_virtualenv and [
if not is_mach_or_build_site and [
s for s in requirement.specifier if s.operator != "=="
]:
raise Exception(
'All virtualenvs except for "mach" and "build" must pin pypi package '
'All sites except for "mach" and "build" must pin pypi package '
f'versions in the format "package==version", found "{raw_requirement}"'
)
return requirement

64
python/mach/mach/virtualenv.py → python/mach/mach/site.py
View File

@ -2,8 +2,8 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This file contains code for populating the virtualenv environment for
# Mozilla's build system. It is typically called as part of configure.
# This file contains code for managing the Python import scope for Mach. This
# generally involves populating a Python virtualenv.
from __future__ import absolute_import, print_function, unicode_literals
@ -23,20 +23,20 @@ PTH_FILENAME = "mach.pth"
METADATA_FILENAME = "moz_virtualenv_metadata.json"
class MozVirtualenvMetadataOutOfDateError(Exception):
class MozSiteMetadataOutOfDateError(Exception):
pass
class MozVirtualenvMetadata:
"""Moz-specific information that is encoded into a file at the root of a virtualenv"""
class MozSiteMetadata:
"""Details about a Moz-managed python environment"""
def __init__(self, hex_version, virtualenv_name, file_path):
def __init__(self, hex_version, site_name, file_path):
self.hex_version = hex_version
self.virtualenv_name = virtualenv_name
self.site_name = site_name
self.file_path = file_path
def write(self):
raw = {"hex_version": self.hex_version, "virtualenv_name": self.virtualenv_name}
raw = {"hex_version": self.hex_version, "virtualenv_name": self.site_name}
with open(self.file_path, "w") as file:
json.dump(raw, file)
@ -44,7 +44,7 @@ class MozVirtualenvMetadata:
return (
type(self) == type(other)
and self.hex_version == other.hex_version
and self.virtualenv_name == other.virtualenv_name
and self.site_name == other.site_name
)
@classmethod
@ -64,8 +64,8 @@ class MozVirtualenvMetadata:
except FileNotFoundError:
return None
except KeyError:
raise MozVirtualenvMetadataOutOfDateError(
f'The virtualenv metadata at "{path}" is out-of-date.'
raise MozSiteMetadataOutOfDateError(
f'The moz site metadata at "{path}" is out-of-date.'
)
@ -95,19 +95,19 @@ class VirtualenvHelper(object):
return os.path.join(self.bin_path, binary)
class VirtualenvManager(VirtualenvHelper):
"""Contains logic for managing virtualenvs for building the tree."""
class MozSiteManager(VirtualenvHelper):
"""Contains logic for managing the Python import scope for building the tree."""
def __init__(
self,
topsrcdir,
virtualenvs_dir,
virtualenv_name,
site_name,
*,
manifest_path=None,
):
virtualenv_path = os.path.join(virtualenvs_dir, virtualenv_name)
super(VirtualenvManager, self).__init__(virtualenv_path)
virtualenv_path = os.path.join(virtualenvs_dir, site_name)
super(MozSiteManager, self).__init__(virtualenv_path)
# __PYVENV_LAUNCHER__ confuses pip, telling it to use the system
# python interpreter rather than the local virtual environment interpreter.
@ -115,19 +115,19 @@ class VirtualenvManager(VirtualenvHelper):
os.environ.pop("__PYVENV_LAUNCHER__", None)
self.topsrcdir = topsrcdir
# Record the Python executable that was used to create the Virtualenv
# Record the Python executable that was used to create the virtualenv
# so we can check this against sys.executable when verifying the
# integrity of the virtualenv.
self.exe_info_path = os.path.join(self.virtualenv_root, "python_exe.txt")
self._virtualenv_name = virtualenv_name
self._site_name = site_name
self._manifest_path = manifest_path or os.path.join(
topsrcdir, "build", f"{virtualenv_name}_virtualenv_packages.txt"
topsrcdir, "build", f"{site_name}_virtualenv_packages.txt"
)
self._metadata = MozVirtualenvMetadata(
self._metadata = MozSiteMetadata(
sys.hexversion,
virtualenv_name,
site_name,
os.path.join(self.virtualenv_root, METADATA_FILENAME),
)
@ -178,12 +178,10 @@ class VirtualenvManager(VirtualenvHelper):
# * If the "hex_version" doesn't match, then the system Python has changed/been
# upgraded.
try:
existing_metadata = MozVirtualenvMetadata.from_path(
self._metadata.file_path
)
existing_metadata = MozSiteMetadata.from_path(self._metadata.file_path)
if existing_metadata != self._metadata:
return False
except MozVirtualenvMetadataOutOfDateError:
except MozSiteMetadataOutOfDateError:
return False
if env_requirements.pth_requirements or env_requirements.vendored_requirements:
@ -207,10 +205,10 @@ class VirtualenvManager(VirtualenvHelper):
return True
def ensure(self):
"""Ensure the virtualenv is present and up to date.
"""Ensure the site is present and up to date.
If the virtualenv is up to date, this does nothing. Otherwise, it
creates and populates the virtualenv as necessary.
If the site is up to date, this does nothing. Otherwise, it
creates and populates a virtualenv as necessary.
This should be the main API used from this class as it is the
highest-level.
@ -258,8 +256,8 @@ class VirtualenvManager(VirtualenvHelper):
def requirements(self):
if not os.path.exists(self._manifest_path):
raise Exception(
f'The current command is using the "{self._virtualenv_name}" '
"virtualenv. However, that virtualenv is missing its associated "
f'The current command is using the "{self._site_name}" '
"site. However, that site is missing its associated "
f'requirements definition file at "{self._manifest_path}".'
)
@ -270,7 +268,7 @@ class VirtualenvManager(VirtualenvHelper):
return MachEnvRequirements.from_requirements_definition(
self.topsrcdir,
is_thunderbird,
self._virtualenv_name in ("mach", "build"),
self._site_name in ("mach", "build"),
self._manifest_path,
)
@ -305,10 +303,10 @@ class VirtualenvManager(VirtualenvHelper):
return self.virtualenv_root
def activate(self):
"""Activate the virtualenv in this Python context.
"""Activate this site in the current Python context.
If you run a random Python script and wish to "activate" the
virtualenv, you can simply instantiate an instance of this class
site, you can simply instantiate an instance of this class
and call .ensure() and .activate() to make the virtualenv active.
"""

2
python/mach/mach/test/python.ini
View File

@ -12,7 +12,7 @@ skip-if = python == 3
skip-if = python == 3
[test_logger.py]
[test_mach.py]
[test_virtualenv_compatibility.py]
[test_site_compatibility.py]
# The Windows and Mac workers only use the internal PyPI mirror,
# which will be missing packages required for this test.
skip-if =

10
python/mach/mach/test/test_virtualenv_compatibility.py → python/mach/mach/test/test_site_compatibility.py
View File

@ -13,7 +13,7 @@ from buildconfig import topsrcdir
from mach.requirements import MachEnvRequirements
def _resolve_command_virtualenv_names():
def _resolve_command_site_names():
virtualenv_names = []
for child in (Path(topsrcdir) / "build").iterdir():
if not child.name.endswith("_virtualenv_packages.txt"):
@ -87,8 +87,8 @@ class PackageCache:
return whl_path
def test_virtualenvs_compatible(tmpdir):
command_virtualenv_names = _resolve_command_virtualenv_names()
def test_sites_compatible(tmpdir):
command_site_names = _resolve_command_site_names()
work_dir = Path(tmpdir)
cache = PackageCache(work_dir)
mach_requirements = _requirement_definition_to_pip_format("mach", cache, True)
@ -109,8 +109,8 @@ def test_virtualenvs_compatible(tmpdir):
]
)
for name in command_virtualenv_names:
print(f'Checking compatibility of "{name}" virtualenv')
for name in command_site_names:
print(f'Checking compatibility of "{name}" site')
command_requirements = _requirement_definition_to_pip_format(
name, cache, name == "build"
)

2
python/mozboot/mozboot/util.py
View File

@ -10,7 +10,7 @@ import platform
import subprocess
from subprocess import CalledProcessError
from mach.virtualenv import VirtualenvHelper
from mach.site import VirtualenvHelper
from mozfile import which

4
python/mozbuild/mozbuild/base.py
View File

@ -289,10 +289,10 @@ class MozbuildObject(ProcessExecutionMixin):
@property
def virtualenv_manager(self):
from mach.virtualenv import VirtualenvManager
from mach.site import MozSiteManager
if self._virtualenv_manager is None:
self._virtualenv_manager = VirtualenvManager(
self._virtualenv_manager = MozSiteManager(
self.topsrcdir,
os.path.join(self.topobjdir, "_virtualenvs"),
self._virtualenv_name,

4
python/mozbuild/mozbuild/mach_commands.py
View File

@ -2458,7 +2458,7 @@ def package_l10n(command_context, verbose=False, locales=[]):
def create_mach_environment(command_context, force=False):
"""Create the mach virtualenv."""
from mozboot.util import get_mach_virtualenv_root
from mach.virtualenv import VirtualenvManager
from mach.site import MozSiteManager
virtualenv_path = get_mach_virtualenv_root()
if sys.executable.startswith(virtualenv_path):
@ -2469,7 +2469,7 @@ def create_mach_environment(command_context, force=False):
)
return 1
manager = VirtualenvManager(
manager = MozSiteManager(
command_context.topsrcdir,
os.path.dirname(virtualenv_path),
"mach",

8
python/mozbuild/mozbuild/sphinx.py
View File

@ -197,7 +197,7 @@ class MozbuildSymbols(Directive):
def setup(app):
from mach.virtualenv import VirtualenvManager
from mach.site import MozSiteManager
from moztreedocs import manager
app.add_directive("mozbuildsymbols", MozbuildSymbols)
@ -214,10 +214,10 @@ def setup(app):
# We need to adjust sys.path in order for Python API docs to get generated
# properly. We leverage the in-tree virtualenv for this.
topsrcdir = manager.topsrcdir
ve = VirtualenvManager(
site = MozSiteManager(
topsrcdir,
os.path.join(app.outdir, "_venv"),
"common",
)
ve.ensure()
ve.activate()
site.ensure()
site.activate()

2
python/mozbuild/mozbuild/test/test_vendor.py
View File

@ -39,7 +39,7 @@ def test_up_to_date_vendor():
work_vendored = os.path.join(work_dir, "third_party", "python")
shutil.copytree(existing_vendored, work_vendored)
# Copy "mach" module so that `VirtualenvManager` can populate itself.
# Copy "mach" module so that `MozSiteManager` can populate itself.
# This is needed because "topsrcdir" is used in this test both for determining
# import paths and for acting as a "work dir".
existing_mach = os.path.join(topsrcdir, "python", "mach")