Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-08 10:44:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Backed out changeset 02cc74634f7d (bug 1798746) for causing build bustages. CLOSED TREE

Browse Source
This commit is contained in:
Iulian Moraru 2023-05-03 20:54:50 +03:00
parent c8052f726d
commit 5f08dc66c3
10 changed files with 91 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
build/mach_initialize.py
View File

@ -503,13 +503,12 @@ def initialize(topsrcdir):
# Sparse checkouts may not have all mach_commands.py files. Ignore
# errors from missing files. Same for spidermonkey tarballs.
repo = resolve_repository()
if repo != "SOURCE":
missing_ok = (
repo is not None and repo.sparse_checkout_present()
) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
else:
missing_ok = ()
missing_ok = (
repo is not None and repo.sparse_checkout_present()
) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
driver.load_commands_from_spec(MACH_COMMANDS, topsrcdir, missing_ok=missing_ok)
return driver

5
python/mozboot/mozboot/base.py
View File

@ -349,10 +349,7 @@ class BaseBootstrapper(object):
def install_toolchain_artifact_impl(
self, install_dir: Path, toolchain_job, no_unpack=False
):
if type(self.srcdir) is str:
mach_binary = Path(self.srcdir) / "mach"
else:
mach_binary = (self.srcdir / "mach").resolve()
mach_binary = (self.srcdir / "mach").resolve()
if not mach_binary.exists():
raise ValueError(f"mach not found at {mach_binary}")

9
python/mozboot/mozboot/bootstrap.py
View File

@ -253,7 +253,7 @@ class Bootstrapper(object):
self.instance = cls(**args)
def maybe_install_private_packages_or_exit(self, application, checkout_type):
def maybe_install_private_packages_or_exit(self, application):
# Install the clang packages needed for building the style system, as
# well as the version of NodeJS that we currently support.
# Also install the clang static-analysis package by default
@ -345,7 +345,7 @@ class Bootstrapper(object):
self._validate_python_environment(checkout_root)
if self.instance.no_system_changes:
self.maybe_install_private_packages_or_exit(application, checkout_type)
self.maybe_install_private_packages_or_exit(application)
self._output_mozconfig(application, mozconfig_builder)
sys.exit(0)
@ -391,7 +391,7 @@ class Bootstrapper(object):
checkout_root,
)
self.maybe_install_private_packages_or_exit(application, checkout_type)
self.maybe_install_private_packages_or_exit(application)
self.check_code_submission(checkout_root)
# Wait until after moz-phab setup to check telemetry so that employees
# will be automatically opted-in.
@ -600,7 +600,6 @@ def current_firefox_checkout(env, hg: Optional[Path] = None):
while path:
hg_dir = path / ".hg"
git_dir = path / ".git"
moz_configure = path / "moz.configure"
if hg and hg_dir.exists():
# Verify the hg repo is a Firefox repo by looking at rev 0.
try:
@ -626,8 +625,6 @@ def current_firefox_checkout(env, hg: Optional[Path] = None):
if moz_configure.exists():
_warn_if_risky_revision(path)
return ("git" if git_dir.exists() else "hg"), path
elif moz_configure.exists():
return "SOURCE", path
if not len(path.parents):
break

30
python/mozbuild/mozbuild/artifact_commands.py
View File

@ -2,8 +2,6 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import argparse
import hashlib
import json
@ -185,12 +183,7 @@ def artifact_install(
no_process=no_process,
)
if source is None and distdir is None:
return artifacts.install_from(
source, distdir or command_context.distdir, src=True
)
else:
return artifacts.install_from(source, distdir or command_context.distdir)
return artifacts.install_from(source, distdir or command_context.distdir)
@ArtifactSubCommand(
@ -440,18 +433,17 @@ def artifact_toolchain(
repo = mozversioncontrol.get_repository_object(
command_context.topsrcdir
)
if not isinstance(repo, mozversioncontrol.SrcRepository):
changed_files = set(repo.get_outgoing_files()) | set(
repo.get_changed_files()
changed_files = set(repo.get_outgoing_files()) | set(
repo.get_changed_files()
)
if changed_files:
command_context.log(
logging.ERROR,
"artifact",
{},
"Hint: consider reverting your local changes "
"to the following files: %s" % sorted(changed_files),
)
if changed_files:
command_context.log(
logging.ERROR,
"artifact",
{},
"Hint: consider reverting your local changes "
"to the following files: %s" % sorted(changed_files),
)
if "TASKCLUSTER_ROOT_URL" in os.environ:
command_context.log(
logging.ERROR,

7
python/mozbuild/mozbuild/base.py
View File

@ -482,12 +482,7 @@ class MozbuildObject(ProcessExecutionMixin):
except InvalidRepoPath:
repo = None
if (
repo
and repo != "SOURCE"
and not vcs_revision
and repo.sparse_checkout_present()
):
if repo and not vcs_revision and repo.sparse_checkout_present():
vcs_revision = "."
if vcs_revision is None:

141
python/mozversioncontrol/mozversioncontrol/__init__.py
View File

@ -50,7 +50,7 @@ class CannotDeleteFromRootOfRepositoryException(Exception):
the repository, which is not permitted."""
def get_tool_path(tool: Optional[Union[str, Path]] = None):
def get_tool_path(tool: Union[str, Path]):
tool = Path(tool)
"""Obtain the path of `tool`."""
if tool.is_absolute() and tool.exists():
@ -78,9 +78,9 @@ class Repository(object):
__metaclass__ = abc.ABCMeta
def __init__(self, path: Path, tool: Optional[str] = None):
def __init__(self, path: Path, tool: str):
self.path = str(path.resolve())
self._tool = Path(get_tool_path(tool)) if tool else None
self._tool = Path(get_tool_path(tool))
self._version = None
self._valid_diff_filter = ("m", "a", "d")
self._env = os.environ.copy()
@ -206,7 +206,7 @@ class Repository(object):
"""Undo the effects of a previous add_remove_files call for `paths`."""
@abc.abstractmethod
def get_tracked_files_finder(self, path=None):
def get_tracked_files_finder(self):
"""Obtain a mozpack.files.BaseFinder of managed files in the working
directory.
@ -495,7 +495,7 @@ class HgRepository(Repository):
self._run("forget", *paths)
def get_tracked_files_finder(self, path=None):
def get_tracked_files_finder(self):
# Can return backslashes on Windows. Normalize to forward slashes.
files = list(
p.replace("\\", "/") for p in self._run("files", "-0").split("\0") if p
@ -672,7 +672,7 @@ class GitRepository(Repository):
self._run("reset", *paths)
def get_tracked_files_finder(self, path=None):
def get_tracked_files_finder(self):
files = [p for p in self._run("ls-files", "-z").split("\0") if p]
return FileListFinder(files)
@ -744,130 +744,7 @@ class GitRepository(Repository):
self._run("config", name, value)
class SrcRepository(Repository):
"""An implementation of `Repository` for Git repositories."""
def __init__(self, path: Path, src="src"):
super(SrcRepository, self).__init__(path, tool=None)
@property
def name(self):
return "src"
@property
def head_ref(self):
pass
@property
def base_ref(self):
pass
def base_ref_as_hg(self):
pass
@property
def branch(self):
pass
@property
def has_git_cinnabar(self):
pass
def get_commit_time(self):
pass
def sparse_checkout_present(self):
pass
def get_user_email(self):
pass
def get_upstream(self):
pass
def get_changed_files(self, diff_filter="ADM", mode="unstaged", rev=None):
pass
def get_outgoing_files(self, diff_filter="ADM", upstream=None):
pass
def add_remove_files(self, *paths: Union[str, Path]):
pass
def forget_add_remove_files(self, *paths: Union[str, Path]):
pass
def git_ignore(self, path):
"""This function reads the mozilla-central/.gitignore file and creates a
list of the patterns to ignore
"""
ignore = []
f = open(path + "/.gitignore", "r")
while True:
line = f.readline()
if not line:
break
if line.startswith("#"):
pass
elif line.strip() and line not in ["\r", "\r\n"]:
ignore.append(line.strip().lstrip("/"))
f.close()
return ignore
def get_files(self, path):
"""This function gets all files in your source folder e.g mozilla-central
and creates a list of that
"""
res = []
# move away the .git or .hg folder from path to more easily test in a hg/git repo
for root, dirs, files in os.walk("."):
for name in files:
res.append(os.path.join(root, name))
return res
def get_tracked_files_finder(self, path):
"""Get files, similar to 'hg files -0' or 'git ls-files -z', thats why
we read the .gitignore file for patterns to ignore.
Speed could probably be improved.
"""
import fnmatch
files = list(
p.replace("\\", "/").replace("./", "") for p in self.get_files(path) if p
)
files.sort()
ig = self.git_ignore(path)
mat = []
for i in ig:
x = fnmatch.filter(files, i)
if x:
mat = mat + x
match = list(set(files) - set(mat))
match.sort()
if len(match) == 0:
return None
else:
return FileListFinder(match)
def working_directory_clean(self, untracked=False, ignored=False):
pass
def clean_directory(self, path: Union[str, Path]):
pass
def update(self, ref):
pass
def push_to_try(self, message, allow_log_capture=False):
pass
def set_config(self, name, value):
pass
def get_repository_object(
path: Optional[Union[str, Path]], hg="hg", git="git", src="src"
):
def get_repository_object(path: Optional[Union[str, Path]], hg="hg", git="git"):
"""Get a repository object for the repository at `path`.
If `path` is not a known VCS repository, raise an exception.
"""
@ -879,8 +756,6 @@ def get_repository_object(
return HgRepository(path, hg=hg)
elif (path / ".git").exists():
return GitRepository(path, git=git)
elif (path / "moz.configure").exists():
return SrcRepository(path, src=src)
else:
raise InvalidRepoPath(f"Unknown VCS, or not a source checkout: {path}")
@ -906,8 +781,6 @@ def get_repository_from_build_config(config):
return HgRepository(Path(config.topsrcdir), hg=config.substs["HG"])
elif flavor == "git":
return GitRepository(Path(config.topsrcdir), git=config.substs["GIT"])
elif flavor == "src":
return SrcRepository(Path(config.topsrcdir), src=config.substs["SRC"])
else:
raise MissingVCSInfo("unknown VCS_CHECKOUT_TYPE value: %s" % flavor)

46
taskcluster/gecko_taskgraph/util/hash.py
View File

@ -21,19 +21,7 @@ def hash_path(path):
@memoize
def get_file_finder(base_path):
from pathlib import Path
repo = get_repository_object(base_path)
if repo:
files = repo.get_tracked_files_finder(base_path)
if files:
return files
else:
return None
else:
return get_repository_object(Path(base_path)).get_tracked_files_finder(
base_path
)
return get_repository_object(base_path).get_tracked_files_finder()
def hash_paths(base_path, patterns):
@ -48,21 +36,19 @@ def hash_paths(base_path, patterns):
finder = get_file_finder(base_path)
h = hashlib.sha256()
files = {}
if finder:
for pattern in patterns:
found = list(finder.find(pattern))
if found:
files.update(found)
else:
raise Exception("%s did not match anything" % pattern)
for path in sorted(files.keys()):
if path.endswith((".pyc", ".pyd", ".pyo")):
continue
h.update(
"{} {}\n".format(
hash_path(mozpath.abspath(mozpath.join(base_path, path))),
mozpath.normsep(path),
).encode("utf-8")
)
for pattern in patterns:
found = list(finder.find(pattern))
if found:
files.update(found)
else:
raise Exception("%s did not match anything" % pattern)
for path in sorted(files.keys()):
if path.endswith((".pyc", ".pyd", ".pyo")):
continue
h.update(
"{} {}\n".format(
hash_path(mozpath.abspath(mozpath.join(base_path, path))),
mozpath.normsep(path),
).encode("utf-8")
)
return h.hexdigest()

2
third_party/python/taskcluster_taskgraph/taskgraph/generator.py vendored
View File

@ -271,7 +271,7 @@ class TaskGraphGenerator:
# Always add legacy target tasks method until we deprecate that API.
if "target_tasks_method" not in filters:
filters.insert(0, "target_tasks_method")
filters = [filter_tasks.filter_task_functions[f] for f in filters if f]
filters = [filter_tasks.filter_task_functions[f] for f in filters]
yield self.verify("parameters", parameters)

121
third_party/python/taskcluster_taskgraph/taskgraph/parameters.py vendored
View File

@ -80,76 +80,46 @@ def get_version(repo_path):
def _get_defaults(repo_root=None):
repo_path = repo_root or os.getcwd()
repo = get_repository(repo_path)
if repo:
try:
repo_url = repo.get_url()
parsed_url = mozilla_repo_urls.parse(repo_url)
project = parsed_url.repo_name
except (
CalledProcessError,
mozilla_repo_urls.errors.InvalidRepoUrlError,
mozilla_repo_urls.errors.UnsupportedPlatformError,
):
repo_url = ""
project = ""
try:
repo_url = repo.get_url()
parsed_url = mozilla_repo_urls.parse(repo_url)
project = parsed_url.repo_name
except (
CalledProcessError,
mozilla_repo_urls.errors.InvalidRepoUrlError,
mozilla_repo_urls.errors.UnsupportedPlatformError,
):
repo_url = ""
project = ""
return {
"base_repository": repo_url,
"base_ref": "",
"base_rev": "",
"build_date": int(time.time()),
"build_number": 1,
"do_not_optimize": [],
"enable_always_target": True,
"existing_tasks": {},
"filters": ["target_tasks_method"],
"head_ref": repo.branch or repo.head_rev,
"head_repository": repo_url,
"head_rev": repo.head_rev,
"head_tag": "",
"level": "3",
"moz_build_date": datetime.now().strftime("%Y%m%d%H%M%S"),
"next_version": None,
"optimize_strategies": None,
"optimize_target_tasks": True,
"owner": "nobody@mozilla.com",
"project": project,
"pushdate": int(time.time()),
"pushlog_id": "0",
"repository_type": repo.tool,
"target_tasks_method": "default",
"tasks_for": "",
"version": get_version(repo_path),
}
else:
return {
"base_repository": "SOURCE",
"base_ref": "",
"base_rev": "",
"build_date": int(time.time()),
"build_number": 1,
"do_not_optimize": [],
"enable_always_target": True,
"existing_tasks": {},
"filters": ["target_tasks_method"],
"head_ref": "",
"head_repository": "",
"head_rev": "",
"head_tag": "",
"level": "3",
"moz_build_date": datetime.now().strftime("%Y%m%d%H%M%S"),
"next_version": None,
"optimize_strategies": None,
"optimize_target_tasks": True,
"owner": "nobody@mozilla.com",
"project": "",
"pushdate": int(time.time()),
"pushlog_id": "0",
"repository_type": "",
"target_tasks_method": "default",
"tasks_for": "",
"version": "",
}
return {
"base_repository": repo_url,
"base_ref": "",
"base_rev": "",
"build_date": int(time.time()),
"build_number": 1,
"do_not_optimize": [],
"enable_always_target": True,
"existing_tasks": {},
"filters": ["target_tasks_method"],
"head_ref": repo.branch or repo.head_rev,
"head_repository": repo_url,
"head_rev": repo.head_rev,
"head_tag": "",
"level": "3",
"moz_build_date": datetime.now().strftime("%Y%m%d%H%M%S"),
"next_version": None,
"optimize_strategies": None,
"optimize_target_tasks": True,
"owner": "nobody@mozilla.com",
"project": project,
"pushdate": int(time.time()),
"pushlog_id": "0",
"repository_type": repo.tool,
"target_tasks_method": "default",
"tasks_for": "",
"version": get_version(repo_path),
}
defaults_functions = [_get_defaults]
@ -225,14 +195,13 @@ class Parameters(ReadOnlyDict):
@staticmethod
def _fill_defaults(repo_root=None, **kwargs):
defaults = {}
if repo_root != "SOURCE":
for fn in defaults_functions:
defaults.update(fn(repo_root))
for fn in defaults_functions:
defaults.update(fn(repo_root))
for name, default in defaults.items():
if name not in kwargs:
kwargs[name] = default
return kwargs
for name, default in defaults.items():
if name not in kwargs:
kwargs[name] = default
return kwargs
def check(self):
schema = (

4
third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py vendored
View File

@ -504,10 +504,8 @@ def get_repository(path):
return HgRepository(path)
elif os.path.exists(os.path.join(path, ".git")):
return GitRepository(path)
elif os.path.exists(os.path.join(path, "moz.configure")):
return None
raise RuntimeError("Current directory is neither a git or hg repository, nor a release source")
raise RuntimeError("Current directory is neither a git or hg repository")
def find_hg_revision_push_info(repository, revision):