RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-02-04 03:06:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
llvm-mirror/utils/git-svn/git-llvm
Zachary Turner aa4a00c53a [git-llvm] Fix some issues surrouding EOL conversion on Windows. 
This patch fixes three issues.

The first is that we didn't consider files which are explicitly
set to eolstyle CRLF in the repo, and there are a handful of
these.

Second is that dos2unix doesn't have a -q option in GnuWin32,
so this codepath wasn't working properly.

Finally with newer versions of Python (or newer versions of Git,
or some combination of the two) patches can't be applied when
we treat stdin as text, because Python silently undoes all the
work we did to convert the newlines to LF using dos2unix by
using universal_newlines=True and then converting them *back*
to CRLF.  So we need to add a way to force stdin to be treated
as binary, and use it when LF-newlines are required.

Differential Revision: https://reviews.llvm.org/D51444

llvm-svn: 344095
2018-10-09 23:42:28 +00:00

377 lines
12 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python
#
# ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
#
# The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
# ==------------------------------------------------------------------------==#
"""
git-llvm integration
====================
This file provides integration for git.
"""
from __future__ import print_function
import argparse
import collections
import contextlib
import errno
import os
import re
import subprocess
import sys
import tempfile
import time
assert sys.version_info >= (2, 7)
# It's *almost* a straightforward mapping from the monorepo to svn...
GIT_TO_SVN_DIR = {
d: (d + '/trunk')
for d in [
'clang-tools-extra',
'compiler-rt',
'debuginfo-tests',
'dragonegg',
'klee',
'libclc',
'libcxx',
'libcxxabi',
'libunwind',
'lld',
'lldb',
'llgo',
'llvm',
'openmp',
'parallel-libs',
'polly',
]
}
GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'})
VERBOSE = False
QUIET = False
dev_null_fd = None
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
def log(*args, **kwargs):
if QUIET:
return
print(*args, **kwargs)
def log_verbose(*args, **kwargs):
if not VERBOSE:
return
print(*args, **kwargs)
def die(msg):
eprint(msg)
sys.exit(1)
def first_dirname(d):
while True:
(head, tail) = os.path.split(d)
if not head or head == '/':
return tail
d = head
def get_dev_null():
"""Lazily create a /dev/null fd for use in shell()"""
global dev_null_fd
if dev_null_fd is None:
dev_null_fd = open(os.devnull, 'w')
return dev_null_fd
def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
ignore_errors=False, force_binary_stdin=False):
log_verbose('Running: %s' % ' '.join(cmd))
err_pipe = subprocess.PIPE
if ignore_errors:
# Silence errors if requested.
err_pipe = get_dev_null()
if force_binary_stdin and stdin:
stdin = stdin.encode('utf-8')
start = time.time()
text = not force_binary_stdin
p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe,
stdin=subprocess.PIPE, universal_newlines=text)
stdout, stderr = p.communicate(input=stdin)
elapsed = time.time() - start
log_verbose('Command took %0.1fs' % elapsed)
if not text:
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
if p.returncode == 0 or ignore_errors:
if stderr and not ignore_errors:
eprint('`%s` printed to stderr:' % ' '.join(cmd))
eprint(stderr.rstrip())
if strip:
stdout = stdout.rstrip('\r\n')
return stdout
err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode)
eprint(err_msg)
if stderr:
eprint(stderr.rstrip())
if die_on_failure:
sys.exit(2)
raise RuntimeError(err_msg)
def git(*cmd, **kwargs):
return shell(['git'] + list(cmd), kwargs.get('strip', True))
def svn(cwd, *cmd, **kwargs):
# TODO: Better way to do default arg when we have *cmd?
return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None),
ignore_errors=kwargs.get('ignore_errors', None))
def program_exists(cmd):
if sys.platform == 'win32' and not cmd.endswith('.exe'):
cmd += '.exe'
for path in os.environ["PATH"].split(os.pathsep):
if os.access(os.path.join(path, cmd), os.X_OK):
return True
return False
def get_default_rev_range():
# Get the branch tracked by the current branch, as set by
# git branch --set-upstream-to See http://serverfault.com/a/352236/38694.
cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
cur_branch)
if not upstream_branch:
upstream_branch = 'origin/master'
# Get the newest common ancestor between HEAD and our upstream branch.
upstream_rev = git('merge-base', 'HEAD', upstream_branch)
return '%s..' % upstream_rev
def get_revs_to_push(rev_range):
if not rev_range:
rev_range = get_default_rev_range()
# Use git show rather than some plumbing command to figure out which revs
# are in rev_range because it handles single revs (HEAD^) and ranges
# (foo..bar) like we want.
revs = git('show', '--reverse', '--quiet',
'--pretty=%h', rev_range).splitlines()
if not revs:
die('Nothing to push: No revs in range %s.' % rev_range)
return revs
def clean_and_update_svn(svn_repo):
svn(svn_repo, 'revert', '-R', '.')
# Unfortunately it appears there's no svn equivalent for git clean, so we
# have to do it ourselves.
for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
if not line.startswith('?'):
continue
filename = line[1:].strip()
os.remove(os.path.join(svn_repo, filename))
svn(svn_repo, 'update', *list(GIT_TO_SVN_DIR.values()))
def svn_init(svn_root):
if not os.path.exists(svn_root):
log('Creating svn staging directory: (%s)' % (svn_root))
os.makedirs(svn_root)
log('This is a one-time initialization, please be patient for a few'
' minutes...')
svn(svn_root, 'checkout', '--depth=immediates',
'https://llvm.org/svn/llvm-project/', '.')
svn(svn_root, 'update', *list(GIT_TO_SVN_DIR.values()))
log("svn staging area ready in '%s'" % svn_root)
if not os.path.isdir(svn_root):
die("Can't initialize svn staging dir (%s)" % svn_root)
def fix_eol_style_native(rev, sr, svn_sr_path):
"""Fix line endings before applying patches with Unix endings
SVN on Windows will check out files with CRLF for files with the
svn:eol-style property set to "native". This breaks `git apply`, which
typically works with Unix-line ending patches. Work around the problem here
by doing a dos2unix up front for files with svn:eol-style set to "native".
SVN will not commit a mass line ending re-doing because it detects the line
ending format for files with this property.
"""
files = git('diff-tree', '--no-commit-id', '--name-only', '-r', rev, '--',
sr).split('\n')
files = [f.split('/', 1)[1] for f in files]
# Skip files that don't exist in SVN yet.
files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
# Use ignore_errors because 'svn propget' prints errors if the file doesn't
# have the named property. There doesn't seem to be a way to suppress that.
eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files,
ignore_errors=True)
crlf_files = []
if len(files) == 1:
# No need to split propget output on ' - ' when we have one file.
if eol_props.strip() in ['native', 'CRLF']:
crlf_files = files
else:
for eol_prop in eol_props.split('\n'):
# Remove spare CR.
eol_prop = eol_prop.strip('\r')
if not eol_prop:
continue
prop_parts = eol_prop.rsplit(' - ', 1)
if len(prop_parts) != 2:
eprint("unable to parse svn propget line:")
eprint(eol_prop)
continue
(f, eol_style) = prop_parts
if eol_style == 'native':
crlf_files.append(f)
if crlf_files:
# Reformat all files with native SVN line endings to Unix format. SVN knows
# files with native line endings are text files. It will commit just the
# diff, and not a mass line ending change.
shell(['dos2unix'] + crlf_files, ignore_errors=True, cwd=svn_sr_path)
def svn_push_one_rev(svn_repo, rev, dry_run):
files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
rev).split('\n')
subrepos = {first_dirname(f) for f in files}
if not subrepos:
raise RuntimeError('Empty diff for rev %s?' % rev)
status = svn(svn_repo, 'status', '--no-ignore')
if status:
die("Can't push git rev %s because svn status is not empty:\n%s" %
(rev, status))
for sr in subrepos:
svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr])
if os.name == 'nt':
fix_eol_style_native(rev, sr, svn_sr_path)
diff = git('show', '--binary', rev, '--', sr, strip=False)
# git is the only thing that can handle its own patches...
log_verbose('Apply patch: %s' % diff)
try:
# If we allow python to apply the diff in text mode, it will silently
# convert \n to \r\n which git doesn't like.
shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff,
die_on_failure=False, force_binary_stdin=True)
except RuntimeError as e:
eprint("Patch doesn't apply: maybe you should try `git pull -r` "
"first?")
sys.exit(2)
status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
for l in (l for l in status_lines if (l.startswith('?') or
l.startswith('I'))):
svn(svn_repo, 'add', '--no-ignore', l[1:].strip())
for l in (l for l in status_lines if l.startswith('!')):
svn(svn_repo, 'remove', l[1:].strip())
# Now we're ready to commit.
commit_msg = git('show', '--pretty=%B', '--quiet', rev)
if not dry_run:
log(svn(svn_repo, 'commit', '-m', commit_msg, '--force-interactive'))
log('Committed %s to svn.' % rev)
else:
log("Would have committed %s to svn, if this weren't a dry run." % rev)
def cmd_push(args):
'''Push changes back to SVN: this is extracted from Justin Lebar's script
available here: https://github.com/jlebar/llvm-repo-tools/
Note: a current limitation is that git does not track file rename, so they
will show up in SVN as delete+add.
'''
# Get the git root
git_root = git('rev-parse', '--show-toplevel')
if not os.path.isdir(git_root):
die("Can't find git root dir")
# Push from the root of the git repo
os.chdir(git_root)
# We need a staging area for SVN, let's hide it in the .git directory.
dot_git_dir = git('rev-parse', '--git-common-dir')
svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn')
svn_init(svn_root)
rev_range = args.rev_range
dry_run = args.dry_run
revs = get_revs_to_push(rev_range)
log('Pushing %d commit%s:\n%s' %
(len(revs), 's' if len(revs) != 1
else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c)
for c in revs)))
for r in revs:
clean_and_update_svn(svn_root)
svn_push_one_rev(svn_root, r, dry_run)
if __name__ == '__main__':
if not program_exists('svn'):
die('error: git-llvm needs svn command, but svn is not installed.')
argv = sys.argv[1:]
p = argparse.ArgumentParser(
prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
description=__doc__)
subcommands = p.add_subparsers(title='subcommands',
description='valid subcommands',
help='additional help')
verbosity_group = p.add_mutually_exclusive_group()
verbosity_group.add_argument('-q', '--quiet', action='store_true',
help='print less information')
verbosity_group.add_argument('-v', '--verbose', action='store_true',
help='print more information')
parser_push = subcommands.add_parser(
'push', description=cmd_push.__doc__,
help='push changes back to the LLVM SVN repository')
parser_push.add_argument(
'-n',
'--dry-run',
dest='dry_run',
action='store_true',
help='Do everything other than commit to svn. Leaves junk in the svn '
'repo, so probably will not work well if you try to commit more '
'than one rev.')
parser_push.add_argument(
'rev_range',
metavar='GIT_REVS',
type=str,
nargs='?',
help="revs to push (default: everything not in the branch's "
'upstream, or not in origin/master if the branch lacks '
'an explicit upstream)')
parser_push.set_defaults(func=cmd_push)
args = p.parse_args(argv)
VERBOSE = args.verbose
QUIET = args.quiet
# Dispatch to the right subcommand
args.func(args)
Reference in New Issue View Git Blame Copy Permalink