Preserve changed filenames and rename sources in tooling checks

Read Git's NUL-delimited UTF-8 output without newline conversion, and
include deleted source paths when files move between check scopes.
Keep pr_ready's line accounting consistent with parity selection.

Fixes #10535

Co-Authored-By: Codex (GPT-6) <[email protected]>

tests/test_pr_ready.py

@@ -2,6 +2,7 @@ import argparse
 import datetime
 import json
 import os
+from pathlib import Path
 import shutil
 import subprocess
 import tempfile
@@ -28,6 +29,70 @@ def args(host="local", quick=False):
     return argparse.Namespace(host=host, quick=quick)
 
 
+class GitChangedPathsTests(unittest.TestCase):
+    def setUp(self):
+        directory = tempfile.TemporaryDirectory()
+        self.addCleanup(directory.cleanup)
+        self.root = Path(directory.name)
+        self.git("init")
+        for key, value in (("core.quotePath", "true"), ("core.autocrlf", "false"),
+                           ("core.hooksPath", str(self.root / "no-hooks")), ("commit.gpgsign", "false")):
+            self.git("config", key, value)
+        for module in (pr_ready, pr_ready.parity):
+            patcher = mock.patch.object(module, "ROOT", self.root)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+
+    def git(self, *arguments):
+        return subprocess.run(["git", *arguments], cwd=self.root, capture_output=True,
+                              text=True, encoding="utf-8", check=True).stdout
+
+    def write(self, path, text):
+        destination = self.root / path
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_text(text, encoding="utf-8", newline="\n")
+
+    def commit(self):
+        self.git("add", ".")
+        self.git("-c", "user.name=Test", "-c", "[email protected]", "commit", "-m", "base")
+
+    def test_unicode_paths_select_checks_and_keep_line_counts(self):
+        tracked = "crates/horizon-core/src/caf\u00e9 edit.rs"
+        untracked = "crates/horizon-core/src/\u65b0 file.rs"
+        self.write(tracked, "first\n")
+        self.commit()
+        self.write(tracked, "first\nsecond\n")
+        self.write(untracked, "new\nfile\n")
+        changed = pr_ready.parity.changed_files("HEAD")
+        self.assertEqual(changed, {tracked, untracked})
+        self.assertEqual(pr_ready.line_counts("HEAD", changed), {tracked: (1, 0), untracked: (2, 0)})
+        self.assertTrue(all(pr_ready.workspace_rust(path) for path in changed))
+        self.assertEqual(pr_ready.touched_crates(changed, PACKAGES), ["horizon-core"])
+
+    def test_renamed_rust_source_keeps_both_paths_and_counts(self):
+        before, after = "crates/horizon-core/src/moved.rs", "docs/moved.md"
+        self.write(before, "fn moved() {\n}\n")
+        self.commit()
+        (self.root / "docs").mkdir()
+        self.git("mv", before, after)
+        changed = pr_ready.parity.changed_files("HEAD")
+        self.assertEqual(changed, {before, after})
+        self.assertEqual(pr_ready.line_counts("HEAD", changed), {before: (0, 2), after: (2, 0)})
+        self.assertEqual(pr_ready.touched_crates(changed, PACKAGES), ["horizon-core"])
+
+    @unittest.skipIf(os.name == "nt", "Windows filenames cannot contain tabs or newlines")
+    def test_tabs_and_newlines_remain_in_paths(self):
+        tracked = "crates/horizon-core/src/tab\tfile.rs"
+        untracked = "crates/horizon-core/src/line\r\nfile.rs"
+        self.write(tracked, "first\n")
+        self.commit()
+        self.write(tracked, "first\nsecond\n")
+        self.write(untracked, "new\n")
+        changed = pr_ready.parity.changed_files("HEAD")
+        self.assertEqual(changed, {tracked, untracked})
+        self.assertEqual(pr_ready.line_counts("HEAD", changed), {tracked: (1, 0), untracked: (1, 0)})
+
+
 class PrReadyTests(unittest.TestCase):
     def test_paths_map_to_the_crates_that_own_them(self):
         self.assertEqual(pr_ready.touched_crates({"crates/horizon-core/src/jmath.rs", "crates/horizon-client/build.rs",

tools/parity.py

@@ -1407,15 +1407,15 @@ def feature_sources_changed(crate, feature, changed):
 
 
 def changed_files(ref):
-    """Files changed between the merge base of HEAD and ref and HEAD (committed or not), or None without a merge base."""
+    """Changed files since the merge base (committed or not), including both sides of renames; None without a base."""
     def git(*arguments):
-        result = subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, text=True)
-        return result.stdout.split("\n") if result.returncode == 0 else None
+        result = subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True)
+        return result.stdout.decode("utf-8") if result.returncode == 0 else None
     base = git("merge-base", "HEAD", ref)
-    if not base or not base[0]:
+    if not base:
         return None
-    return set(filter(None, git("diff", "--name-only", base[0]) or [])) | \
-        set(filter(None, git("ls-files", "--others", "--exclude-standard") or []))
+    return set(filter(None, (git("diff", "--name-only", "--no-renames", "-z", base.strip()) or "").split("\0"))) | \
+        set(filter(None, (git("ls-files", "--others", "--exclude-standard", "-z") or "").split("\0")))
 
 
 def canonical_dependency_path(path: str) -> str:

tools/pr_ready.py

@@ -189,17 +189,17 @@ def duration(seconds):
 
 
 def git(*arguments):
-    return subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, text=True, check=True).stdout
+    return subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, check=True).stdout.decode("utf-8")
 
 
 def line_counts(base, changed):
     """Lines added and removed in each changed path since base, untracked files included: {path: (added, removed)}."""
     counts = {}
-    for row in git("diff", "--numstat", base).splitlines():
+    for row in filter(None, git("diff", "--numstat", "--no-renames", "-z", base).split("\0")):
         plus, minus, path = row.split("\t", 2)
         if plus != "-":
             counts[path] = (int(plus), int(minus))
-    for path in git("ls-files", "--others", "--exclude-standard").splitlines():
+    for path in git("ls-files", "--others", "--exclude-standard", "-z").split("\0"):
         if path in changed:
             try:
                 counts[path] = (len((ROOT / path).read_text(encoding="utf-8").splitlines()), 0)