Setup, Control and Terminal: a child's non-UTF-8 output (GBK adapter names) no longer kills Step 1 or launch (#4675)

setup.py

@@ -906,7 +906,7 @@ def run(cmd, cwd=None, env=None, check=True, quiet=False):
     env.setdefault("STRATA_EXIT_WITH_PARENT", "1")     # tools/mtp_fetch.py ends when setup ends (#3257: a cancel left it writing)
     r = subprocess.run([str(c) for c in cmd], cwd=cwd, env=env,
                        stdout=subprocess.PIPE if quiet else None, stderr=subprocess.STDOUT if quiet else None,
-                       text=True)
+                       text=True, errors="replace")
     if check and r.returncode != 0:
         if quiet and r.stdout:
             say(r.stdout[-4000:])
@@ -935,7 +935,7 @@ def fetch_mtp(mtp: Path, env: dict, tries: int = 3):
 
 def out(cmd):
     try:
-        return subprocess.run(cmd, capture_output=True, text=True, timeout=60).stdout
+        return subprocess.run(cmd, capture_output=True, text=True, errors="replace", timeout=60).stdout or ""
     except (OSError, subprocess.TimeoutExpired):
         return ""
 
@@ -975,7 +975,7 @@ def ram_gb():
     if sys.platform == "darwin":
         try:
             value = subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True,
-                                   text=True, timeout=3, check=True).stdout.strip()
+                                   text=True, errors="replace", timeout=3, check=True).stdout.strip()
             return int(value) / 2**30
         except (OSError, ValueError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
             return 0.0
@@ -2215,7 +2215,7 @@ CMAKE_MIN = (3, 24)                    # CMakeLists.txt: cmake_minimum_required(
 def _tool_version(path, name):
     """(ok, version tuple) of `path --version`: ok False when it does not run (a pip launcher whose Python is gone)."""
     try:
-        done = subprocess.run([path, "--version"], capture_output=True, text=True, timeout=30)
+        done = subprocess.run([path, "--version"], capture_output=True, text=True, errors="replace", timeout=30)
     except (OSError, subprocess.SubprocessError):
         return False, ()
     if done.returncode != 0:
@@ -4209,11 +4209,12 @@ _DISPLAY_CLASS = r"SYSTEM\CurrentControlSet\Control\Class\{4d36e968-e325-11ce-bf
 
 def _win_display_adapters() -> list[dict]:
     """The display adapters present ({"name", "pnp"}), from Win32_VideoController."""
-    ps = ("Get-CimInstance Win32_VideoController | ForEach-Object { $_.Name + '|' + $_.PNPDeviceID + '|' + "
+    ps = ("[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; "      # the pipe is read as UTF-8 (#4675)
+          "Get-CimInstance Win32_VideoController | ForEach-Object { $_.Name + '|' + $_.PNPDeviceID + '|' + "
           "$_.AdapterRAM }")
     text = out(["powershell", "-NoProfile", "-NonInteractive", "-Command", ps])
     found = []
-    for line in text.splitlines():
+    for line in (text or "").splitlines():
         parts = line.strip().split("|")
         if len(parts) >= 2 and parts[1]:
             ram = parts[2] if len(parts) > 2 else ""
@@ -4285,7 +4286,8 @@ def amd_gpus_windows(adapters=None, registry=None) -> list[dict]:
         for k, r in enumerate(registry):               # the same card's driver instance: its 64-bit VRAM size
             if k in used or _pci_device_id(r.get("MatchingDeviceId")) != did:
                 continue
-            if r.get("DriverDesc") and ad.get("name") and r["DriverDesc"].strip() != ad["name"].strip():
+            if (r.get("DriverDesc") and ad.get("name") and "\ufffd" not in ad["name"]
+                    and r["DriverDesc"].strip() != ad["name"].strip()):
                 continue
             used.add(k)
             mem = r.get("HardwareInformation.qwMemorySize")
@@ -4353,7 +4355,7 @@ def hip_devices(probe: Path | None = None, text: str | None = None) -> list[dict
             return None
         try:
             hip_runtime_beside_exe(probe.parent)       # #468 #461: not the driver's System32 copy
-            r = subprocess.run([str(probe), "--list-devices"], capture_output=True, text=True, timeout=120,
+            r = subprocess.run([str(probe), "--list-devices"], capture_output=True, text=True, errors="replace", timeout=120,
                                cwd=str(probe.parent), env=engine_env(probe.parent))
         except (OSError, subprocess.TimeoutExpired):
             return None
@@ -4479,7 +4481,7 @@ def engine_from_feed(eng: Path, updating: bool = False) -> Path | None:
     say("  Fetching the ready-made engine from the signed update feed ...")
     try:
         run = subprocess.run([sys.executable, str(ROOT / "strata.py"), "update", "--no-control", "--output", "json"],
-                             env=env, cwd=str(ROOT), capture_output=True, text=True, timeout=3600)
+                             env=env, cwd=str(ROOT), capture_output=True, text=True, errors="replace", timeout=3600)
     except (OSError, subprocess.SubprocessError) as e:
         warn(f"the update feed could not be run ({e})")
         return None
@@ -7406,7 +7408,7 @@ def mtp_corrupt(mtp: Path, env=None) -> bool:
     if not (mtp / "tensors").is_dir():
         return False
     r = subprocess.run([sys.executable, str(ROOT / "tools" / "mtp_fetch.py"), "verify", "--out", str(mtp)], env=env,
-                       stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+                       stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors="replace")
     return r.returncode == 3
 
 
@@ -7686,7 +7688,7 @@ def engine_step(a, gpu, hip, amd, cuda_tk, vision):
 def engine_families(eng: Path) -> set:
     """The model families an engine runs (`strata --families`; an engine from before it: Flash-Next's only)."""
     try:
-        text = subprocess.run([str(eng / EXE), "--families"], capture_output=True, text=True, timeout=60,
+        text = subprocess.run([str(eng / EXE), "--families"], capture_output=True, text=True, errors="replace", timeout=60,
                               env=engine_env(eng)).stdout      # #2372: with the engine's rocm/bin on PATH
     except (OSError, subprocess.SubprocessError):
         return {"qwen4exp"}

strata_cli/cli.py

@@ -1575,7 +1575,7 @@ def _timeskip(a: argparse.Namespace) -> int:
         env = {**os.environ, "STRATA_TIMESKIP_PROMPT": prompt,
                "STRATA_TIMESKIP_ATTEMPT": str(attempt)}
         proc = subprocess.run(a.revise_command, shell=True, cwd=str(candidate), env=env,
-                              capture_output=True, text=True)
+                              capture_output=True, text=True, errors="replace")
         if proc.returncode != 0:
             raise time_skip.TimeSkipError(
                 f"the revision command failed (exit {proc.returncode}): "

strata_cli/launch/access.py

@@ -103,7 +103,7 @@ def cloudflared_version(path: str) -> str | None:
     if key in _cf_seen:
         return _cf_seen[key]
     try:
-        r = subprocess.run([path, "--version"], capture_output=True, text=True, timeout=10,
+        r = subprocess.run([path, "--version"], capture_output=True, text=True, errors="replace", timeout=10,
                            creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0) if WINDOWS else 0)
         text = (r.stdout or "") + (r.stderr or "")
     except (OSError, subprocess.SubprocessError):
@@ -145,13 +145,13 @@ def cloudflared_procs(exclude: tuple[int, ...] = ()) -> list[tuple[int, str]]:
         if WINDOWS:
             text = subprocess.run(["powershell.exe", "-NoProfile", "-NonInteractive", "-Command",
                                    "Get-CimInstance Win32_Process -Filter \"Name='cloudflared.exe'\" | ForEach-Object { \"$($_.ProcessId)`t$($_.CommandLine)\" }"],
-                                  capture_output=True, text=True, timeout=15, creationflags=subprocess.CREATE_NO_WINDOW).stdout
+                                  capture_output=True, text=True, errors="replace", timeout=15, creationflags=subprocess.CREATE_NO_WINDOW).stdout
             for line in text.splitlines():
                 m = re.match(r"(\d+)\t(.*)$", line)
                 if m:
                     out.append((int(m.group(1)), m.group(2)))
         else:
-            text = subprocess.run(["ps", "-axo", "pid=,comm=,args="], capture_output=True, text=True, timeout=5).stdout
+            text = subprocess.run(["ps", "-axo", "pid=,comm=,args="], capture_output=True, text=True, errors="replace", timeout=5).stdout
             for line in text.splitlines():
                 m = re.match(r"\s*(\d+)\s+(\S+)\s*(.*)$", line)
                 if m and re.search(r"(^|/)cloudflared$", m.group(2)):
@@ -185,7 +185,7 @@ def find_tailscale() -> str | None:
 
 def _run(argv: list[str], timeout: float = 8.0) -> str:
     try:
-        r = subprocess.run(argv, capture_output=True, text=True, timeout=timeout)
+        r = subprocess.run(argv, capture_output=True, text=True, errors="replace", timeout=timeout)
         return r.stdout or ""
     except (OSError, subprocess.SubprocessError):
         return ""

strata_cli/launch/params.py

@@ -256,7 +256,7 @@ def dsv41_help_text(engine: str | Path | None) -> str:
     if engine:
         import subprocess
         try:
-            proc = subprocess.run([str(engine), "--help"], capture_output=True, text=True, timeout=20)
+            proc = subprocess.run([str(engine), "--help"], capture_output=True, text=True, errors="replace", timeout=20)
             seen = set(re.findall(r"--[a-z][a-z0-9-]*", (proc.stdout or "") + " " + (proc.stderr or "")))
             lines += [f"  {f}  An engine setting (the engine's help has no sentence for it)." for f in sorted(seen) if f not in DSV41_FLAGS]
         except Exception:                                    # noqa: BLE001 - the static list stands
@@ -269,7 +269,7 @@ def engine_help_text(engine: str | Path | None) -> str | None:
     import subprocess
     try:
         from strata_control import tuning
-        proc = subprocess.run([*tuning.engine_command(Path(engine)), "--help"], capture_output=True, text=True, timeout=20,
+        proc = subprocess.run([*tuning.engine_command(Path(engine)), "--help"], capture_output=True, text=True, errors="replace", timeout=20,
                               env=tuning.engine_env(Path(engine)))
     except Exception:                                        # noqa: BLE001
         return None

strata_cli/launch/runner.py

@@ -415,7 +415,7 @@ def dsv41_engine_version(exe) -> tuple:
     if version:
         return version
     try:
-        done = subprocess.run([str(exe), "--version"], capture_output=True, text=True, timeout=10, stdin=subprocess.DEVNULL)
+        done = subprocess.run([str(exe), "--version"], capture_output=True, text=True, errors="replace", timeout=10, stdin=subprocess.DEVNULL)
         found = re.search(r"(\d+\.\d+\.\d+(?:\.\d+)?)", (done.stdout or "") + (done.stderr or ""))
         return tuple(int(x) for x in found.group(1).split(".")) if found else ()
     except (OSError, subprocess.SubprocessError, ValueError):

strata_control/doctor.py

@@ -86,7 +86,7 @@ def _nvidia_cards() -> tuple[list[dict] | None, str | None]:
     try:
         result = subprocess.run(
             [tool, "--query-gpu=index,name,uuid,memory.total,driver_version",
-             "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5, check=False,
+             "--format=csv,noheader,nounits"], capture_output=True, text=True, errors="replace", timeout=5, check=False,
         )
     except (OSError, UnicodeError, subprocess.TimeoutExpired):
         return None, "nvidia-smi could not complete within five seconds"

strata_control/dsv41_engine.py

@@ -213,7 +213,7 @@ def _test_start(exe: Path, timeout: float = 120.0) -> tuple[bool, str]:
     The package's own folders go on the library path first: a Windows HIP package carries the ROCm runtime it loads in rocm/bin."""
     import subprocess
     try:
-        done = subprocess.run([str(exe), "--version"], capture_output=True, text=True, timeout=timeout, cwd=str(Path(exe).parent),
+        done = subprocess.run([str(exe), "--version"], capture_output=True, text=True, errors="replace", timeout=timeout, cwd=str(Path(exe).parent),
                               env=start_env(exe))
     except (OSError, subprocess.SubprocessError) as exc:
         return False, f"it could not be started ({exc})"

strata_control/engine_component.py

@@ -475,11 +475,11 @@ def detect_backend(environ=None) -> tuple[str | None, str]:
         if not tool:
             continue
         try:
-            done = subprocess.run([tool, *command[1:]], capture_output=True, timeout=8, text=True,
+            done = subprocess.run([tool, *command[1:]], capture_output=True, timeout=8, text=True, errors="replace",
                                   creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
         except (OSError, subprocess.SubprocessError):
             continue
-        if done.returncode == 0 and done.stdout.strip():
+        if done.returncode == 0 and (done.stdout or "").strip():
             if backend == "hip" and "gfx" not in done.stdout:
                 continue
             if backend == "sycl" and "level_zero" not in done.stdout.lower() and "gpu" not in done.stdout.lower():
@@ -630,13 +630,13 @@ def _windows_adapters_cim() -> list[dict]:
         return []
     try:
         done = subprocess.run([shell, "-NoProfile", "-NonInteractive", "-Command",
-                               "Get-CimInstance Win32_VideoController | ForEach-Object { $_.PNPDeviceID + '|' + $_.Name + '|' + $_.AdapterRAM }"],
-                              capture_output=True, text=True, timeout=15, stdin=subprocess.DEVNULL,
+                               "[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; Get-CimInstance Win32_VideoController | ForEach-Object { $_.PNPDeviceID + '|' + $_.Name + '|' + $_.AdapterRAM }"],
+                              capture_output=True, text=True, errors="replace", timeout=15, stdin=subprocess.DEVNULL,
                               creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
     except (OSError, subprocess.SubprocessError):
         return []
     found = []
-    for line in done.stdout.splitlines():
+    for line in (done.stdout or "").splitlines():
         match = re.search(r"VEN_([0-9A-F]{4})&DEV_([0-9A-F]{4})", line, re.I)
         if not match:
             continue
@@ -773,10 +773,10 @@ def _smi_query(field_name: str) -> list[str]:
         return []
     try:
         done = subprocess.run([tool, f"--query-gpu={field_name}", "--format=csv,noheader"], capture_output=True,
-                              timeout=8, text=True, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
+                              timeout=8, text=True, errors="replace", creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
     except (OSError, subprocess.SubprocessError):
         return []
-    return [line.strip() for line in done.stdout.splitlines() if line.strip()] if done.returncode == 0 else []
+    return [line.strip() for line in (done.stdout or "").splitlines() if line.strip()] if done.returncode == 0 and done.stdout else []
 
 
 def nvidia_driver_version() -> str | None:

strata_control/gpu_roles.py

@@ -451,7 +451,7 @@ def nvidia_speeds(timeout: float = 20.0) -> dict[int, float]:
         env = dict(os.environ)
         env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
         env.pop("CUDA_VISIBLE_DEVICES", None)
-        r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=timeout, env=env)
+        r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, errors="replace", timeout=timeout, env=env)
         out = {}
         for line in r.stdout.splitlines():
             i, sm, khz = (int(x) for x in line.split())

strata_control/tuning.py

@@ -212,7 +212,7 @@ def engine_flag_inventory(engine: Path) -> list[str] | None:
     at all.
     """
     try:
-        proc = subprocess.run([*engine_command(engine), "--help"], capture_output=True, text=True, timeout=20,
+        proc = subprocess.run([*engine_command(engine), "--help"], capture_output=True, text=True, errors="replace", timeout=20,
                               env=engine_env(engine))
     except (OSError, subprocess.SubprocessError):
         return None

strata_control/updater.py

@@ -229,7 +229,7 @@ def _gfx_names() -> list[str]:
         tool = shutil.which("rocminfo") if os.name != "nt" else shutil.which("hipInfo")
         if tool:
             try:
-                done = subprocess.run([tool], capture_output=True, text=True, timeout=10,
+                done = subprocess.run([tool], capture_output=True, text=True, errors="replace", timeout=10,
                                       creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
                 found = sorted(set(re.findall(r"\bgfx[0-9a-f]{3,4}\b", done.stdout)))
             except (OSError, subprocess.SubprocessError):
@@ -767,7 +767,7 @@ def binary_smoke(component: ec.EngineComponent, timeout: float = 60) -> dict:
     env = dict(os.environ)
     env[var] = os.pathsep.join(ec.package_lib_dirs(component.root)) + os.pathsep + env.get(var, "")
     try:
-        done = subprocess.run([str(component.exe), "--version"], capture_output=True, text=True, timeout=timeout, env=env,
+        done = subprocess.run([str(component.exe), "--version"], capture_output=True, text=True, errors="replace", timeout=timeout, env=env,
                               cwd=str(component.root), creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
     except (OSError, subprocess.SubprocessError) as exc:
         raise TestStartFailed(component.version, f"the engine could not run ({type(exc).__name__}: {exc})") from None
@@ -1305,7 +1305,7 @@ def _control_version(interpreter: str, entry: Path, stage: Path, ctx: Context, v
     done = None
     for attempt, timeout in enumerate(CONTROL_VERSION_TIMEOUTS):
         try:
-            done = subprocess.run([interpreter, str(entry), "--version"], capture_output=True, text=True, timeout=timeout, cwd=str(stage),
+            done = subprocess.run([interpreter, str(entry), "--version"], capture_output=True, text=True, errors="replace", timeout=timeout, cwd=str(stage),
                                   env={**ctx.environ, "PYTHONNOUSERSITE": "1", "STRATA_NO_ACQUIRE": "1", "STRATA_HANDOVER": "1"},
                                   creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
             last_error = None

tools/strata_mcp.py

@@ -345,7 +345,7 @@ def spawn_detached(cmd: list, cwd: Path, log_path: Path, env: dict | None = None
 def run_quiet(cmd, timeout=30) -> str:
     """A command's stdout ('' when it is missing or fails), without a console window flashing up on Windows."""
     try:
-        return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL,
+        return subprocess.run(cmd, capture_output=True, text=True, errors="replace", timeout=timeout, stdin=subprocess.DEVNULL,
                               creationflags=CREATE_NO_WINDOW if WIN else 0).stdout or ""
     except (OSError, subprocess.SubprocessError, ValueError):
         return ""

tools/test_setup_gbk_adapters.py

@@ -0,0 +1,44 @@
+"""#4675: a child process that writes non-UTF-8 bytes (a GBK display adapter name) must not kill Step 1.
+
+    python -m unittest tools.test_setup_gbk_adapters
+"""
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+import setup  # noqa: E402
+
+GBK_LINE = b"Microsoft \xbb\xf9\xb1\xbe\xcf\xd4\xca\xbe\xca\xca\xc5\xe4\xc6\xf7|PCI/VEN_10DE&DEV_0A60&SUBSYS|0\r\n"
+
+
+class GbkOutput(unittest.TestCase):
+    def test_out_survives_bytes_that_are_not_utf8(self):
+        code = "import sys; sys.stdout.buffer.write(%r)" % (GBK_LINE,)
+        text = setup.out([sys.executable, "-X", "utf8", "-c", code])
+        self.assertIsInstance(text, str)
+        self.assertIn("Microsoft", text)
+        self.assertIn("VEN_10DE", text)
+
+    def test_adapter_list_never_gets_none(self):
+        from unittest import mock
+        with mock.patch.object(setup, "out", return_value=None):
+            self.assertEqual(setup._win_display_adapters(), [])
+        with mock.patch.object(setup, "out", return_value=GBK_LINE.decode("utf-8", "replace")):
+            found = setup._win_display_adapters()
+        self.assertEqual(len(found), 1)
+        self.assertIn("VEN_10DE", found[0]["pnp"])
+
+    def test_a_mangled_name_still_matches_its_registry_entry_by_pci_id(self):
+        adapters = [{"name": "AMD Radeon(TM) Graphics �", "pnp": "PCI/VEN_1002&DEV_744C&SUBSYS", "ram": 0}]
+        registry = [{"DriverDesc": "AMD Radeon RX 7900 XTX", "MatchingDeviceId": "PCI/VEN_1002&DEV_744C",
+                     "qwMemorySize": 24 << 30}]
+        g = setup.amd_gpus_windows(adapters, registry)
+        self.assertEqual(len(g), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()