mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-24 05:11:16 +00:00
a34741e58d
Also fix update-angle.py: * Use shell=True to get ninja to run * Don't record "/PDBSourcePath:" because it depends on configuration of the vendoring machine and is otherwise unused, and so uselessly causes blame noise. Differential Revision: https://phabricator.services.mozilla.com/D171824
560 lines
15 KiB
Python
Executable File
560 lines
15 KiB
Python
Executable File
#! /usr/bin/env python3
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
assert __name__ == "__main__"
|
|
|
|
"""
|
|
To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and
|
|
python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come
|
|
before python3 in your path.
|
|
|
|
Upstream: https://chromium.googlesource.com/angle/angle
|
|
|
|
Our repo: https://github.com/mozilla/angle
|
|
It has branches like 'firefox-60' which is the branch we use for pulling into
|
|
Gecko with this script.
|
|
|
|
This script leaves a record of the merge-base and cherry-picks that we pull into
|
|
Gecko. (gfx/angle/cherries.log)
|
|
|
|
ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/
|
|
An easy choice is to grab Chrome's Beta's ANGLE branch.
|
|
|
|
## Usage
|
|
|
|
Prepare your env:
|
|
|
|
~~~
|
|
export PATH="$PATH:/path/to/depot_tools"
|
|
~~~
|
|
|
|
If this is a new repo, don't forget:
|
|
|
|
~~~
|
|
# In the angle repo:
|
|
./scripts/bootstrap.py
|
|
gclient sync
|
|
~~~
|
|
|
|
Update: (in the angle repo)
|
|
|
|
~~~
|
|
# In the angle repo:
|
|
/path/to/gecko/gfx/angle/update-angle.py origin
|
|
git push moz # Push the firefox-XX branch to github.com/mozilla/angle
|
|
~~~~
|
|
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from typing import * # mypy annotations
|
|
|
|
REPO_DIR = pathlib.Path.cwd()
|
|
GECKO_ANGLE_DIR = pathlib.Path(__file__).parent
|
|
|
|
OUT_DIR = pathlib.Path("out")
|
|
|
|
COMMON_HEADER = [
|
|
"# Generated by update-angle.py",
|
|
"",
|
|
'include("../../moz.build.common")',
|
|
]
|
|
|
|
ROOTS = ["//:translator", "//:libEGL", "//:libGLESv2"]
|
|
|
|
CHECK_ONLY = False
|
|
args = sys.argv[1:]
|
|
while True:
|
|
arg = args.pop(0)
|
|
if arg == "--check":
|
|
CHECK_ONLY = True
|
|
continue
|
|
args.insert(0, arg)
|
|
break
|
|
|
|
GN_ENV = dict(os.environ)
|
|
GN_ENV["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0"
|
|
|
|
(GIT_REMOTE,) = args # Not always 'origin'!
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
|
def run_checked(*args, **kwargs):
|
|
print(" ", args)
|
|
sys.stdout.flush()
|
|
return subprocess.run(args, check=True, **kwargs)
|
|
|
|
|
|
def sorted_items(x):
|
|
for k in sorted(x.keys()):
|
|
yield (k, x[k])
|
|
|
|
|
|
def collapse_dotdots(path):
|
|
split = path.split("/")
|
|
|
|
ret = []
|
|
for x in split:
|
|
if x == ".." and ret:
|
|
ret.pop()
|
|
continue
|
|
ret.append(x)
|
|
continue
|
|
|
|
return "/".join(ret)
|
|
|
|
|
|
def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]):
|
|
visited_keys: Set[str] = set()
|
|
|
|
def recurse(key):
|
|
if key in visited_keys:
|
|
return
|
|
visited_keys.add(key)
|
|
|
|
t = pre_recurse_func(key)
|
|
try:
|
|
(next_keys, post_recurse_func) = t
|
|
except ValueError:
|
|
(next_keys,) = t
|
|
post_recurse_func = None
|
|
|
|
for x in next_keys:
|
|
recurse(x)
|
|
|
|
if post_recurse_func:
|
|
post_recurse_func(key)
|
|
return
|
|
|
|
for x in root_keys:
|
|
recurse(x)
|
|
return
|
|
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
print("Importing graph")
|
|
|
|
# shutil.rmtree(str(OUT_DIR), True)
|
|
OUT_DIR.mkdir(exist_ok=True)
|
|
|
|
GN_ARGS = b"""
|
|
# Build arguments go here.
|
|
# See "gn args <out_dir> --list" for available build arguments.
|
|
is_clang = true
|
|
is_debug = false
|
|
angle_build_all = false
|
|
angle_enable_abseil = false
|
|
angle_enable_apple_translator_workarounds = true
|
|
angle_enable_essl = true
|
|
angle_enable_gl = false
|
|
angle_enable_gl_desktop_frontend = false
|
|
angle_enable_glsl = true
|
|
angle_enable_null = false
|
|
angle_enable_share_context_lock = true
|
|
angle_enable_vulkan = false
|
|
angle_has_astc_encoder = false
|
|
use_custom_libcxx = false
|
|
"""[
|
|
1:
|
|
]
|
|
args_gn_path = OUT_DIR / "args.gn"
|
|
args_gn_path.write_bytes(GN_ARGS)
|
|
|
|
try:
|
|
run_checked("gn", "gen", str(OUT_DIR), shell=True, env=GN_ENV)
|
|
except subprocess.CalledProcessError:
|
|
sys.stderr.buffer.write(b"`gn` failed. Is depot_tools in your PATH?\n")
|
|
exit(1)
|
|
|
|
p = run_checked(
|
|
"python3",
|
|
"scripts/export_targets.py",
|
|
str(OUT_DIR),
|
|
*ROOTS,
|
|
stdout=subprocess.PIPE,
|
|
shell=True,
|
|
env=GN_ENV,
|
|
)
|
|
|
|
# -
|
|
|
|
print("\nProcessing graph")
|
|
libraries = json.loads(p.stdout.decode())
|
|
|
|
# -
|
|
# HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different
|
|
# platforms.
|
|
# descs["//:angle_common"]["sources"] +=
|
|
EXTRA_ANGLE_COMMON_SOURCES = [
|
|
"//src/common/system_utils_apple.cpp",
|
|
"//src/common/system_utils_linux.cpp",
|
|
"//src/common/system_utils_mac.cpp",
|
|
"//src/common/system_utils_posix.cpp",
|
|
]
|
|
|
|
angle_common = libraries["//:angle_common"]
|
|
angle_common["sources"] += EXTRA_ANGLE_COMMON_SOURCES
|
|
angle_common["sources"] = sorted(angle_common["sources"])
|
|
|
|
# -
|
|
# Reuse our own zlib
|
|
|
|
del libraries["//third_party/zlib:zlib"]
|
|
|
|
# -
|
|
|
|
if CHECK_ONLY:
|
|
print("\n--check complete.")
|
|
exit(0)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Output to moz.builds
|
|
|
|
import vendor_from_git
|
|
|
|
print("")
|
|
vendor_from_git.record_cherry_picks(GECKO_ANGLE_DIR, GIT_REMOTE)
|
|
|
|
# --
|
|
|
|
|
|
def sortedi(x):
|
|
return sorted(x, key=str.lower)
|
|
|
|
|
|
def append_arr(dest, name, vals, indent=0):
|
|
if not vals:
|
|
return
|
|
|
|
dest.append("{}{} += [".format(" " * 4 * indent, name))
|
|
for x in sortedi(vals):
|
|
dest.append('{}"{}",'.format(" " * 4 * (indent + 1), x))
|
|
dest.append("{}]".format(" " * 4 * indent))
|
|
dest.append("")
|
|
return
|
|
|
|
|
|
REGISTERED_DEFINES = {
|
|
"ADLER32_SIMD_SSSE3": False,
|
|
"ANGLE_CAPTURE_ENABLED": True,
|
|
"ANGLE_DISABLE_POOL_ALLOC": True,
|
|
"ANGLE_EGL_LIBRARY_NAME": False,
|
|
"ANGLE_ENABLE_APPLE_WORKAROUNDS": True,
|
|
"ANGLE_ENABLE_D3D11": True,
|
|
"ANGLE_ENABLE_D3D11_COMPOSITOR_NATIVE_WINDOW": True,
|
|
"ANGLE_ENABLE_D3D9": True,
|
|
"ANGLE_ENABLE_DEBUG_ANNOTATIONS": True,
|
|
"ANGLE_ENABLE_NULL": False,
|
|
"ANGLE_ENABLE_OPENGL": False,
|
|
"ANGLE_ENABLE_OPENGL_NULL": False,
|
|
"ANGLE_ENABLE_ESSL": True,
|
|
"ANGLE_ENABLE_GLSL": True,
|
|
"ANGLE_ENABLE_HLSL": True,
|
|
"ANGLE_ENABLE_SHARE_CONTEXT_LOCK": True,
|
|
"ANGLE_GENERATE_SHADER_DEBUG_INFO": True,
|
|
"ANGLE_GLESV2_LIBRARY_NAME": True,
|
|
"ANGLE_HAS_ASTCENC": True,
|
|
"ANGLE_HAS_VULKAN_SYSTEM_INFO": False,
|
|
"ANGLE_IS_64_BIT_CPU": False,
|
|
"ANGLE_IS_WIN": False,
|
|
"ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES": False,
|
|
"ANGLE_SHARED_LIBVULKAN": True,
|
|
"ANGLE_USE_CUSTOM_LIBVULKAN": True,
|
|
"ANGLE_USE_EGL_LOADER": True,
|
|
"ANGLE_VK_LAYERS_DIR": True,
|
|
"ANGLE_VK_MOCK_ICD_JSON": True,
|
|
"ANGLE_VMA_VERSION": True,
|
|
"ASTCENC_DECOMPRESS_ONLY": True,
|
|
"CERT_CHAIN_PARA_HAS_EXTRA_FIELDS": False,
|
|
"CHROMIUM_BUILD": False,
|
|
"COMPONENT_BUILD": False,
|
|
"CRC32_SIMD_SSE42_PCLMUL": False,
|
|
"DEFLATE_FILL_WINDOW_SSE2": False,
|
|
"DYNAMIC_ANNOTATIONS_ENABLED": True,
|
|
"EGL_EGL_PROTOTYPES": True,
|
|
"EGL_EGLEXT_PROTOTYPES": True,
|
|
"EGLAPI": True,
|
|
"FIELDTRIAL_TESTING_ENABLED": False,
|
|
"FULL_SAFE_BROWSING": False,
|
|
"GL_API": True,
|
|
"GL_APICALL": True,
|
|
"GL_GLES_PROTOTYPES": True,
|
|
"GL_GLEXT_PROTOTYPES": True,
|
|
"GPU_INFO_USE_SETUPAPI": True,
|
|
"INFLATE_CHUNK_READ_64LE": False,
|
|
"INFLATE_CHUNK_SIMD_SSE2": False,
|
|
"LIBANGLE_IMPLEMENTATION": True,
|
|
"LIBEGL_IMPLEMENTATION": True,
|
|
"LIBGLESV2_IMPLEMENTATION": True,
|
|
"NOMINMAX": True,
|
|
"NO_TCMALLOC": False,
|
|
# Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object
|
|
"NTDDI_VERSION": False,
|
|
"PSAPI_VERSION": False,
|
|
"SAFE_BROWSING_CSD": False,
|
|
"SAFE_BROWSING_DB_LOCAL": False,
|
|
"UNICODE": True,
|
|
"USE_AURA": False,
|
|
"V8_DEPRECATION_WARNINGS": False,
|
|
"VK_USE_PLATFORM_WIN32_KHR": False,
|
|
"WIN32": False,
|
|
"WIN32_LEAN_AND_MEAN": False,
|
|
"WINAPI_FAMILY": False,
|
|
"WINVER": True,
|
|
# Otherwise:
|
|
# gfx/angle/targets/libANGLE
|
|
# In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10:
|
|
# In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17:
|
|
# C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO'
|
|
# _In_ const POINTER_INFO& pointerInfo
|
|
# ^
|
|
"WTF_USE_DYNAMIC_ANNOTATIONS": False,
|
|
"X86_WINDOWS": False,
|
|
"ZLIB_IMPLEMENTATION": True,
|
|
"_ATL_NO_OPENGL": True,
|
|
"_CRT_NONSTDC_NO_DEPRECATE": True,
|
|
"_CRT_NONSTDC_NO_WARNINGS": True,
|
|
"_CRT_RAND_S": True,
|
|
"_CRT_SECURE_NO_DEPRECATE": True,
|
|
"_DEBUG": False,
|
|
"_HAS_EXCEPTIONS": True,
|
|
"_HAS_ITERATOR_DEBUGGING": False,
|
|
"_SCL_SECURE_NO_DEPRECATE": True,
|
|
"_SECURE_ATL": True,
|
|
"_UNICODE": True,
|
|
"_USING_V110_SDK71_": False,
|
|
"_WIN32_WINNT": False,
|
|
"_WINDOWS": False,
|
|
"_WINSOCK_DEPRECATED_NO_WARNINGS": True,
|
|
"__STD_C": False,
|
|
# clang specific
|
|
"CR_CLANG_REVISION": True,
|
|
"NDEBUG": False,
|
|
"NVALGRIND": False,
|
|
"_HAS_NODISCARD": False,
|
|
}
|
|
|
|
# -
|
|
|
|
print("\nRun actions")
|
|
required_files: Set[str] = set()
|
|
required_files.add("//LICENSE")
|
|
|
|
run_checked("ninja", "-C", str(OUT_DIR), ":angle_commit_id", shell=True)
|
|
required_files.add("//out/gen/angle/angle_commit.h")
|
|
|
|
# -
|
|
|
|
# Export our targets
|
|
print("\nExport targets")
|
|
|
|
# Clear our dest directories
|
|
targets_dir = pathlib.Path(GECKO_ANGLE_DIR, "targets")
|
|
checkout_dir = pathlib.Path(GECKO_ANGLE_DIR, "checkout")
|
|
|
|
shutil.rmtree(targets_dir, True)
|
|
shutil.rmtree(checkout_dir, True)
|
|
targets_dir.mkdir(exist_ok=True)
|
|
checkout_dir.mkdir(exist_ok=True)
|
|
|
|
# -
|
|
|
|
RE_TARGET_NAME = re.compile("//(.*):(.+)")
|
|
|
|
|
|
def export_target(target_full_name) -> Set[str]:
|
|
# print(' ', target_full_name)
|
|
descs = libraries
|
|
desc = descs[target_name]
|
|
flat = desc
|
|
|
|
m = RE_TARGET_NAME.match(target_name)
|
|
assert m, target_name
|
|
name = m.group(2)
|
|
|
|
required_files: Set[str] = set(flat["sources"])
|
|
|
|
# Create our manifest lines
|
|
target_dir = targets_dir / name
|
|
target_dir.mkdir(exist_ok=True)
|
|
|
|
lines = list(COMMON_HEADER)
|
|
lines.append("")
|
|
|
|
for x in sorted(set(desc["defines"])):
|
|
try:
|
|
(k, v) = x.split("=", 1)
|
|
if v.startswith('"'):
|
|
v = f"'{v}'"
|
|
else:
|
|
v = f'"{v}"'
|
|
except ValueError:
|
|
(k, v) = (x, "True")
|
|
|
|
line = f'DEFINES["{k}"] = {v}'
|
|
try:
|
|
if REGISTERED_DEFINES[k] == False:
|
|
line = "# " + line
|
|
except KeyError:
|
|
print(f"[{name}] Unrecognized define: {k}")
|
|
line = "# Unrecognized: " + line
|
|
lines.append(line)
|
|
lines.append("")
|
|
|
|
cxxflags = set(desc["cflags"] + desc["cflags_cc"])
|
|
|
|
def fixup_paths(listt):
|
|
for x in set(listt):
|
|
assert x.startswith("//"), x
|
|
yield "../../checkout/" + x[2:]
|
|
|
|
sources_by_config: Dict[str, List[str]] = {}
|
|
extras: Dict[str, str] = dict()
|
|
for x in fixup_paths(flat["sources"]):
|
|
# print(' '*5, x)
|
|
(b, e) = x.rsplit(".", 1)
|
|
if e in ["h", "hpp", "y", "l", "inc", "inl"]:
|
|
continue
|
|
elif e in ["cpp", "cc", "c"]:
|
|
if b.endswith("_win") or b.endswith("_win32"):
|
|
config = 'CONFIG["OS_ARCH"] == "WINNT"'
|
|
elif b.endswith("_linux"):
|
|
# Include these on BSDs too.
|
|
config = 'CONFIG["OS_ARCH"] not in ("Darwin", "WINNT")'
|
|
elif b.endswith("_apple") or b.endswith("_mac"):
|
|
config = 'CONFIG["OS_ARCH"] == "Darwin"'
|
|
elif b.endswith("_posix"):
|
|
config = 'CONFIG["OS_ARCH"] != "WINNT"'
|
|
else:
|
|
config = "" # None can't compare against str.
|
|
|
|
sources_by_config.setdefault(config, []).append(x)
|
|
continue
|
|
elif e == "rc":
|
|
assert "RCFILE" not in extras, (target_name, extras["RCFILE"], x)
|
|
extras["RCFILE"] = f'"{x}"'
|
|
continue
|
|
elif e == "def":
|
|
assert "DEFFILE" not in extras, (target_name, extras["DEFFILE"], x)
|
|
extras["DEFFILE"] = f'"{x}"'
|
|
continue
|
|
else:
|
|
assert False, ("Unhandled ext:", x)
|
|
|
|
ldflags = set(desc["ldflags"])
|
|
DEF_PREFIX = "/DEF:"
|
|
for x in set(ldflags):
|
|
if x.startswith(DEF_PREFIX):
|
|
def_path = x[len(DEF_PREFIX) :]
|
|
required_files.add(def_path)
|
|
assert "DEFFILE" not in extras
|
|
ldflags.remove(x)
|
|
|
|
def_path = str(OUT_DIR) + "/" + def_path
|
|
def_path = "//" + collapse_dotdots(def_path)
|
|
|
|
def_rel_path = list(fixup_paths([def_path]))[0]
|
|
extras["DEFFILE"] = '"{}"'.format(def_rel_path)
|
|
elif x.startswith("/PDBSourcePath:"):
|
|
ldflags.remove(x)
|
|
|
|
os_libs = list(map(lambda x: x[: -len(".lib")], set(desc.get("libs", []))))
|
|
|
|
def append_arr_commented(dest, name, src):
|
|
lines = []
|
|
append_arr(lines, name, src)
|
|
|
|
def comment(x):
|
|
if x:
|
|
x = "# " + x
|
|
return x
|
|
|
|
lines = map(comment, lines)
|
|
dest += lines
|
|
|
|
append_arr(lines, "LOCAL_INCLUDES", fixup_paths(desc["include_dirs"]))
|
|
append_arr_commented(lines, "CXXFLAGS", cxxflags)
|
|
|
|
for (config, v) in sorted_items(sources_by_config):
|
|
indent = 0
|
|
if config:
|
|
lines.append("if {}:".format(config))
|
|
indent = 1
|
|
append_arr(lines, "SOURCES", v, indent=indent)
|
|
|
|
dep_libs: Set[str] = set()
|
|
for dep_full_name in set(flat["dep_libs"]):
|
|
assert dep_full_name.startswith("//"), dep_name
|
|
(_, dep_name) = dep_full_name.split(":")
|
|
dep_libs.add(dep_name)
|
|
|
|
dep_dirs = set(dep_libs)
|
|
dep_dirs.discard("zlib")
|
|
|
|
append_arr(lines, "USE_LIBS", dep_libs)
|
|
append_arr(lines, "DIRS", ["../" + x for x in dep_dirs])
|
|
append_arr(lines, "OS_LIBS", os_libs)
|
|
append_arr_commented(lines, "LDFLAGS", ldflags)
|
|
|
|
for (k, v) in sorted(extras.items()):
|
|
lines.append("{} = {}".format(k, v))
|
|
|
|
lib_type = desc["type"]
|
|
if lib_type == "shared_library":
|
|
lines.append(f'GeckoSharedLibrary("{name}", linkage=None)')
|
|
elif lib_type == "static_library":
|
|
lines.append(f'Library("{name}")')
|
|
else:
|
|
assert False, lib_type
|
|
|
|
lines.append("")
|
|
# EOF newline
|
|
|
|
# Write it out
|
|
|
|
mozbuild = target_dir / "moz.build"
|
|
print(" ", " ", f"Writing {mozbuild}")
|
|
data = b"\n".join((x.encode() for x in lines))
|
|
mozbuild.write_bytes(data)
|
|
|
|
return required_files
|
|
|
|
|
|
# -
|
|
|
|
for target_name in libraries:
|
|
reqs = export_target(target_name)
|
|
required_files |= reqs
|
|
|
|
# Copy all the files
|
|
|
|
print("\nMigrate required files")
|
|
|
|
i = 0
|
|
for x in required_files:
|
|
i += 1
|
|
sys.stdout.write(f"\r Copying {i}/{len(required_files)}")
|
|
sys.stdout.flush()
|
|
assert x.startswith("//"), x
|
|
x = x[2:]
|
|
|
|
src = REPO_DIR / x
|
|
dest = checkout_dir / x
|
|
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
data = src.read_bytes()
|
|
data = data.replace(b"\r\n", b"\n")
|
|
dest.write_bytes(data)
|
|
|
|
print("\n\nDone")
|