Bug 1566336 - Build clang from git rather than subversion. r=glandium

Differential Revision: https://phabricator.services.mozilla.com/D38361

MANUAL PUSH: avoid closing autoland while clang rebuilds.
This commit is contained in:
Sylvestre Ledru 2019-08-01 05:51:09 +09:00 committed by Mike Hommey
parent f17b58a4c2
commit 2505df426c
17 changed files with 68 additions and 136 deletions

View File

@ -8,11 +8,11 @@
"unpack": true
},
{
"version": "SVN 1.9.4, repacked from SlikSvn (https://sliksvn.com/download/)",
"size": 3934520,
"digest": "d3b8f74936857ecbf542e403ed6835938a31d65302985729cbfa7191bf2cf94138565cefcc2f31517098013fbfc51868348863a55b588250902f9dec214dbc42",
"version": "MinGit-2.13.3-64-bit",
"size": 21482885,
"digest": "929bb3c07be8487ee519422a312bdbfeec8f4db4b62c49d02f9aad9fd2a66c0ee5fad63d2b06c8744c336dc9d50446fa4457897333ad17ffd783ecabd1e2ddbb",
"algorithm": "sha512",
"filename": "svn194.zip",
"filename": "git.zip",
"unpack": true
},
{

View File

@ -20,6 +20,8 @@ from distutils.dir_util import copy_tree
from mozfile import which
URL_REPO = "https://github.com/llvm/llvm-project"
def symlink(source, link_name):
os_symlink = getattr(os, "symlink", None)
@ -76,7 +78,7 @@ def import_clang_tidy(source_dir):
clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]),
'..', 'clang-plugin')
clang_tidy_path = os.path.join(source_dir,
'tools/clang/tools/extra/clang-tidy')
'clang-tools-extra/clang-tidy')
sys.path.append(clang_plugin_path)
from import_mozilla_checks import do_import
do_import(clang_plugin_path, clang_tidy_path)
@ -197,13 +199,14 @@ def install_asan_symbols(build_dir, clang_dir):
shutil.copy2(src_path[0], dst_path[0])
def svn_co(source_dir, url, directory, revision):
run_in(source_dir, ["svn", "co", "-q", "-r", revision, url, directory])
def git_clone(base_dir, url, directory, revision):
run_in(base_dir, ["git", "clone", "-n", url, directory])
run_in(os.path.join(base_dir, directory), ["git", "checkout", revision])
def svn_update(directory, revision):
run_in(directory, ["svn", "revert", "-q", "-R", "."])
run_in(directory, ["svn", "update", "-q", "-r", revision])
def git_update(directory, revision):
run_in(directory, ["git", "remote", "update"])
run_in(directory, ["git", "reset", "--hard", revision])
def is_darwin():
@ -582,8 +585,8 @@ if __name__ == "__main__":
os.sys.exit(0)
llvm_source_dir = source_dir + "/llvm"
extra_source_dir = source_dir + "/clang-tools-extra"
clang_source_dir = source_dir + "/clang"
extra_source_dir = source_dir + "/extra"
lld_source_dir = source_dir + "/lld"
compiler_rt_source_dir = source_dir + "/compiler-rt"
libcxx_source_dir = source_dir + "/libcxx"
@ -605,14 +608,8 @@ if __name__ == "__main__":
config = json.load(args.config)
llvm_revision = config["llvm_revision"]
llvm_repo = config["llvm_repo"]
clang_repo = config["clang_repo"]
extra_repo = config.get("extra_repo")
lld_repo = config.get("lld_repo")
# On some packages we don't use compiler_repo
compiler_repo = config.get("compiler_repo")
libcxx_repo = config["libcxx_repo"]
libcxxabi_repo = config.get("libcxxabi_repo")
if not re.match(r'^[0-9a-fA-F]{40}$', llvm_revision):
raise ValueError("Incorrect format of the git revision")
stages = 3
if "stages" in config:
stages = int(config["stages"])
@ -686,26 +683,15 @@ if __name__ == "__main__":
if not os.path.exists(source_dir):
os.makedirs(source_dir)
def checkout_or_update(repo, checkout_dir):
if os.path.exists(checkout_dir):
svn_update(checkout_dir, llvm_revision)
else:
svn_co(source_dir, repo, checkout_dir, llvm_revision)
if not args.skip_checkout:
checkout_or_update(llvm_repo, llvm_source_dir)
checkout_or_update(clang_repo, clang_source_dir)
if compiler_repo is not None:
checkout_or_update(compiler_repo, compiler_rt_source_dir)
checkout_or_update(libcxx_repo, libcxx_source_dir)
if lld_repo:
checkout_or_update(lld_repo, lld_source_dir)
if libcxxabi_repo:
checkout_or_update(libcxxabi_repo, libcxxabi_source_dir)
if extra_repo:
checkout_or_update(extra_repo, extra_source_dir)
for p in config.get("patches", []):
patch(p, source_dir)
if os.path.exists(os.path.join(source_dir, '.git')):
git_update(source_dir, llvm_revision)
else:
delete(source_dir)
git_clone(base_dir, URL_REPO, source_dir, llvm_revision)
for p in config.get("patches", []):
patch(p, source_dir)
compiler_rt_source_link = llvm_source_dir + "/projects/compiler-rt"
@ -731,7 +717,7 @@ if __name__ == "__main__":
package_name = "clang"
if build_clang_tidy:
package_name = "clang-tidy"
import_clang_tidy(llvm_source_dir)
import_clang_tidy(source_dir)
if not os.path.exists(build_dir):
os.makedirs(build_dir)

View File

@ -1,14 +1,9 @@
{
"llvm_revision": "305830",
"llvm_revision": "449c3ef93afc7a668eb35e67a83717453e28b25a",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_401/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_401/final",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_401/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_401/final",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_401/final",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",

View File

@ -1,15 +1,9 @@
{
"llvm_revision": "349247",
"llvm_revision": "d0d8eb2e5415b8be29343e3c17a18e49e67b5551",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_701/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_701/final",
"lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_701/final",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_701/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_701/final",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_701/final",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",

View File

@ -1,15 +1,9 @@
{
"llvm_revision": "356365",
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "2",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_800/final",
"lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_800/final",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_800/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_800/final",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_800/final",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",

View File

@ -1,15 +1,9 @@
{
"llvm_revision": "356365",
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_800/final",
"lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_800/final",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_800/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_800/final",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_800/final",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",

View File

@ -1,15 +1,9 @@
{
"llvm_revision": "356365",
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_800/final",
"lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_800/final",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_800/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_800/final",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_800/final",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",

View File

@ -1,16 +1,10 @@
{
"llvm_revision": "356365",
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "1",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"osx_cross_compile": true,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_800/final",
"lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_800/final",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_800/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_800/final",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_800/final",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/clang/bin/clang",

View File

@ -1,15 +1,9 @@
{
"llvm_revision": "356265",
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_800/final",
"lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_800/final",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_800/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_800/final",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_800/final",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",

View File

@ -1,5 +1,5 @@
--- a/extra/clang-tidy/tool/run-clang-tidy.py 2019-03-27 15:12:48.000000000 +0200
+++ b/extra/clang-tidy/tool/run-clang-tidy.py 2019-03-27 15:12:39.000000000 +0200
--- a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py 2019-03-27 15:12:48.000000000 +0200
+++ b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py 2019-03-27 15:12:39.000000000 +0200
@@ -169,6 +169,7 @@
with lock:
sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8') + '\n')

View File

@ -1,15 +1,10 @@
{
"llvm_revision": "356365",
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "1",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"build_clang_tidy": true,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final/",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_800/final/",
"extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/tags/RELEASE_800/final/",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_800/final/",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_800/final/",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",

View File

@ -1,16 +1,11 @@
{
"llvm_revision": "356365",
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "1",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"build_clang_tidy": true,
"osx_cross_compile": true,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_800/final",
"extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/tags/RELEASE_800/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_800/final",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_800/final",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/clang/bin/clang",
@ -21,6 +16,7 @@
"libtool": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin-libtool",
"ld": "/builds/worker/workspace/build/src/clang/bin/clang",
"patches": [
"clang-tidy-8.patch"
"clang-tidy-8.patch",
"compiler-rt-no-codesign.patch"
]
}

View File

@ -1,15 +1,10 @@
{
"llvm_revision": "356365",
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "1",
"build_libcxx": false,
"build_type": "Release",
"assertions": false,
"build_clang_tidy": true,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_800/final",
"extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/tags/RELEASE_800/final",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_800/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_800/final",
"python_path": "c:/mozilla-build/python/python.exe",
"cc": "cl.exe",
"cxx": "cl.exe",

View File

@ -1,14 +1,9 @@
{
"llvm_revision": "356365",
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "3",
"build_libcxx": false,
"build_type": "Release",
"assertions": false,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_800/final",
"lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_800/final",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_800/final",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_800/final",
"python_path": "c:/mozilla-build/python/python.exe",
"cc": "cl.exe",
"cxx": "cl.exe",

View File

@ -104,8 +104,11 @@ def write_third_party_paths(mozilla_path, module_path):
def do_import(mozilla_path, clang_tidy_path):
module = 'mozilla'
module_path = os.path.join(clang_tidy_path, module)
if not os.path.isdir(module_path):
os.mkdir(module_path)
try:
os.makedirs(module_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
copy_dir_contents(mozilla_path, module_path)
write_third_party_paths(mozilla_path, module_path)

View File

@ -49,8 +49,8 @@ prepare() {
mkdir -p $TOOLCHAIN_DIR
touch $TOOLCHAIN_DIR/.build-clang
mkdir -p $SRC_DIR
pushd $SRC_DIR
mkdir -p $TOOLCHAIN_DIR
pushd $TOOLCHAIN_DIR
git clone -n git://git.code.sf.net/p/mingw-w64/mingw-w64
pushd mingw-w64
@ -99,32 +99,34 @@ EOF
build_mingw() {
mkdir mingw-w64-headers
pushd mingw-w64-headers
$SRC_DIR/mingw-w64/mingw-w64-headers/configure --host=$machine-w64-mingw32 \
--enable-sdk=all \
--enable-idl \
--with-default-msvcrt=ucrt \
--with-default-win32-winnt=$default_win32_winnt \
--prefix=$CROSS_PREFIX_DIR
$TOOLCHAIN_DIR/mingw-w64/mingw-w64-headers/configure \
--host=$machine-w64-mingw32 \
--enable-sdk=all \
--enable-idl \
--with-default-msvcrt=ucrt \
--with-default-win32-winnt=$default_win32_winnt \
--prefix=$CROSS_PREFIX_DIR
make $make_flags install
popd
mkdir mingw-w64-crt
pushd mingw-w64-crt
$SRC_DIR/mingw-w64/mingw-w64-crt/configure --host=$machine-w64-mingw32 \
$crt_flags \
--with-default-msvcrt=ucrt \
CC="$CC" \
AR=llvm-ar \
RANLIB=llvm-ranlib \
DLLTOOL=llvm-dlltool \
--prefix=$CROSS_PREFIX_DIR
$TOOLCHAIN_DIR/mingw-w64/mingw-w64-crt/configure \
--host=$machine-w64-mingw32 \
$crt_flags \
--with-default-msvcrt=ucrt \
CC="$CC" \
AR=llvm-ar \
RANLIB=llvm-ranlib \
DLLTOOL=llvm-dlltool \
--prefix=$CROSS_PREFIX_DIR
make $make_flags
make $make_flags install
popd
mkdir widl
pushd widl
$SRC_DIR/mingw-w64/mingw-w64-tools/widl/configure --target=$machine-w64-mingw32 --prefix=$INSTALL_DIR
$TOOLCHAIN_DIR/mingw-w64/mingw-w64-tools/widl/configure --target=$machine-w64-mingw32 --prefix=$INSTALL_DIR
make $make_flags
make $make_flags install
popd
@ -193,7 +195,7 @@ build_libcxx() {
-DLIBUNWIND_ENABLE_CROSS_UNWINDING=FALSE \
-DCMAKE_CXX_FLAGS="${DEBUG_FLAGS} -Wno-dll-attribute-on-redeclaration -nostdinc++ -I$SRC_DIR/libcxx/include -DPSAPI_VERSION=2" \
-DCMAKE_C_FLAGS="-Wno-dll-attribute-on-redeclaration" \
$SRC_DIR/libunwind
$TOOLCHAIN_DIR/libunwind
make $make_flags
make $make_flags install
popd
@ -275,7 +277,7 @@ build_utils() {
ln -s llvm-strip $machine-w64-mingw32-strip
ln -s llvm-readobj $machine-w64-mingw32-readobj
ln -s llvm-objcopy $machine-w64-mingw32-objcopy
./clang $SRC_DIR/llvm-mingw/wrappers/windres-wrapper.c -O2 -Wl,-s -o $machine-w64-mingw32-windres
./clang $TOOLCHAIN_DIR/llvm-mingw/wrappers/windres-wrapper.c -O2 -Wl,-s -o $machine-w64-mingw32-windres
popd
}

View File

@ -34,7 +34,8 @@ export PATH="${VSWINPATH}/VC/bin/Hostx64/x64:${VSWINPATH}/SDK/bin/10.0.17134.0/x
export INCLUDE="${VSWINPATH}/VC/include:${VSWINPATH}/VC/atlmfc/include:${VSWINPATH}/SDK/Include/10.0.17134.0/ucrt:${VSWINPATH}/SDK/Include/10.0.17134.0/shared:${VSWINPATH}/SDK/Include/10.0.17134.0/um:${VSWINPATH}/SDK/Include/10.0.17134.0/winrt:${VSWINPATH}/DIA SDK/include"
export LIB="${VSWINPATH}/VC/lib/x64:${VSWINPATH}/VC/atlmfc/lib/x64:${VSWINPATH}/SDK/Lib/10.0.17134.0/ucrt/x64:${VSWINPATH}/SDK/Lib/10.0.17134.0/um/x64:${VSWINPATH}/DIA SDK/lib/amd64"
export PATH="$(cd svn && pwd)/bin:${PATH}"
# Add git.exe to the path
export PATH="$(pwd)/cmd:${PATH}"
export PATH="$(cd cmake && pwd)/bin:${PATH}"
export PATH="$(cd ninja && pwd)/bin:${PATH}"