Server: gpu_roles (Main last, Off excluded, hand-edited gpu wins), roles log line, every-card-Off handling

serve/server.py

@@ -70,6 +70,7 @@ from strata_control import hipblaslt_tuning  # noqa: E402
 from strata_control import vision_reader  # noqa: E402
 from strata_control import draft_layer  # noqa: E402
 from strata_control import user_args  # noqa: E402
+from strata_control import gpu_roles  # noqa: E402
 from serve import ready as ready_file  # noqa: E402
 from serve import owner_guard  # noqa: E402
 from serve import launch_identity  # noqa: E402
@@ -2812,13 +2813,50 @@ class Vision:
 def gpu_list(cfg: dict) -> list[int]:
     """The config's "gpu": one card (2), or several for a layer split ([0, 2] or "0,2"), numbered as nvidia-smi
     numbers them; [] when it names none."""
-    g = cfg.get("gpu")
+    roles, _ = gpu_roles.saved_roles(cfg)       # "gpu_roles" (Main / Helper / Off per card) wins unless "gpu" was edited by hand
+    if roles is not None:
+        legacy = _gpu_numbers(cfg.get("gpu"))
+        return legacy if legacy else gpu_roles.engine_order(roles)     # Main LAST; Off cards left out; [] when every card is Off
+    return _gpu_numbers(cfg.get("gpu"))
+
+
+def _gpu_numbers(g) -> list[int]:
     if g is None or g == "":
         return []
     items = g if isinstance(g, (list, tuple)) else str(g).split(",")
     return [int(str(x).strip()) for x in items if str(x).strip() != ""]
 
 
+def gpu_roles_active(cfg: dict) -> bool:
+    """True when the run settings carry usable "gpu_roles": the card order is the person's (Main last), never re-sorted by speed."""
+    return gpu_roles.saved_roles(cfg)[0] is not None
+
+
+def gpu_all_off(cfg: dict) -> bool:
+    """Every card is set to Off ("gpu_roles" with no card on, or "gpu_none": true)."""
+    roles, _ = gpu_roles.saved_roles(cfg)
+    return roles is not None and not any(r != "off" for r in roles.values())
+
+
+def card_roles_lines(cfg: dict) -> list[str]:
+    """The plain lines printed at start: the roles (cards numbered as nvidia-smi numbers them), a hand-edited "gpu" that beat them, and the
+    all-Off case.  SystemExit when every card is Off and the engine cannot start without one (the CUDA, HIP and SYCL engines)."""
+    roles, problem = gpu_roles.saved_roles(cfg)
+    out = [f"[strata] {problem}"] if problem else []
+    if roles is None:
+        return out
+    if gpu_all_off(cfg):
+        if cfg.get("backend") in ("cpu", "metal") or sys.platform == "darwin":
+            out.append("[strata] every graphics card is Off: " + gpu_roles.CPU_ONLY_LINE)
+            return out
+        raise SystemExit("[strata] every graphics card is set to Off, and this engine cannot start without one. Turn a card on "
+                         "(Main or Helper) in the app or with strata launch --gpu-roles.")
+    word = {"main": "Main", "helper": "Helper", "off": "Off"}
+    out.append("[strata] card roles: " + ", ".join(f"{word[r]} GPU {i}" for r in ("main", "helper", "off")
+                                                    for i in sorted(roles) if roles[i] == r) + " (numbers as nvidia-smi gives them)")
+    return out
+
+
 def engine_binary(exe: str, cfg: dict | None = None) -> str:
     """The file that holds the engine's own option strings, for a feature check that reads it.  The Intel launcher
     (sycl/serve/strata-sycl.sh) is a shell script that runs its repo's STRATA_SYCL_BIN (default build-sycl-aot/strata)
@@ -3109,7 +3147,7 @@ def ordered_gpus(cfg: dict, scores=None, vrams=None) -> list[int]:
     draft and verify have a floor the smaller card cannot afford.  scores/vrams: {index: value} (tests); None asks
     the driver - injected scores without injected vrams means no VRAM check (a test seam is a test seam)."""
     gl = gpu_list(cfg)
-    if len(gl) < 2 or cfg.get("gpu_order") == "as_given":
+    if len(gl) < 2 or cfg.get("gpu_order") == "as_given" or gpu_roles_active(cfg):
         return gl
     args = cfg.get("args") or []
     if "--peer-device" in args or "--layer-split" in args or layer_split_value(cfg) != "auto":
@@ -3189,7 +3227,7 @@ def hip_auto_devices(cfg: dict, env: dict) -> list[int] | None:
     """HIP_VISIBLE_DEVICES for a start that names no card (see pick_hip_devices), asked of the engine package's own `strata-device`.
     None leaves everything as it was: a card the config or the environment names, no probe next to the engine, or a probe that fails.
     SystemExit with the plain line when no card of this computer can run."""
-    if cfg.get("backend") != "hip" or gpu_list(cfg) or str(cfg.get("hip_ordinal") if cfg.get("hip_ordinal") is not None else "").strip() \
+    if cfg.get("backend") != "hip" or gpu_list(cfg) or gpu_all_off(cfg) or str(cfg.get("hip_ordinal") if cfg.get("hip_ordinal") is not None else "").strip() \
             or env.get("HIP_VISIBLE_DEVICES") or env.get("ROCR_VISIBLE_DEVICES"):
         return None
     exe = cfg.get("exe")
@@ -3220,7 +3258,10 @@ def child_env(cfg: dict, *, offline: bool = False) -> dict:
     """The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
     compiled it) first on the library search path."""
     env = dict(os.environ)
-    if hip_visible(cfg) and cfg.get("backend") == "hip":   # AMD: numbered as HIP numbers them (hip_visible)
+    if gpu_all_off(cfg):                             # every card Off: the engine sees none (a CPU engine runs; the others say so at start)
+        env["CUDA_VISIBLE_DEVICES"] = ""
+        env["HIP_VISIBLE_DEVICES"] = ""
+    elif hip_visible(cfg) and cfg.get("backend") == "hip":   # AMD: numbered as HIP numbers them (hip_visible)
         env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in hip_visible(cfg))
     elif gpu_list(cfg):                              # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
         env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"      # own default order (fastest first) can number the cards otherwise
@@ -7974,6 +8015,8 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
         print("model unloaded; the first request loads it ..." if lazy else
               "loading the model (the first start takes a minute or two) ...", flush=True)
         warn_budget_over_ram(engine_args(cfg) if "args" in cfg else [])      # #1080
+        for line in card_roles_lines(cfg):
+            print(line, flush=True)
         note = card_note(cfg, env)
         if note:
             print(note, flush=True)

serve/test_gpu_roles_server.py

@@ -0,0 +1,63 @@
+"""serve/server.py: the run settings' "gpu_roles" (Main / Helper / Off) become the engine's card list, Main LAST, Off left out."""
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from serve.server import card_roles_lines, child_env, gpu_list, ordered_gpus  # noqa: E402
+
+
+class Roles(unittest.TestCase):
+    def test_roles_alone_give_the_order_main_last_and_off_left_out(self):
+        cfg = {"gpu_roles": {"0": "helper", "1": "main", "2": "off"}, "args": []}
+        self.assertEqual(gpu_list(cfg), [0, 1])
+        self.assertEqual(gpu_list({"gpu_roles": {"0": "main", "1": "helper"}, "args": []}), [1, 0])
+        env = child_env({"gpu_roles": {"0": "main", "1": "helper", "2": "off"}, "args": []})
+        self.assertEqual(env["CUDA_VISIBLE_DEVICES"], "1,0")
+
+    def test_the_mirror_is_used_and_never_re_sorted_by_speed(self):
+        cfg = {"gpu_roles": {"0": "main", "1": "helper"}, "gpu": [1, 0], "gpu_order": "as_given", "args": []}
+        self.assertEqual(ordered_gpus(cfg, {0: 1.0, 1: 9.0}), [1, 0])             # the faster card is Helper on purpose
+        cfg2 = {"gpu_roles": {"0": "main", "1": "helper"}, "gpu": [1, 0], "args": []}      # even without gpu_order
+        self.assertEqual(ordered_gpus(cfg2, {0: 1.0, 1: 9.0}), [1, 0])
+
+    def test_one_card_on(self):
+        cfg = {"gpu_roles": {"0": "off", "1": "main"}, "gpu": 1, "gpu_order": "as_given", "args": []}
+        self.assertEqual(gpu_list(cfg), [1])
+        self.assertEqual(child_env(cfg)["CUDA_VISIBLE_DEVICES"], "1")
+
+    def test_a_hand_edited_gpu_wins(self):
+        cfg = {"gpu_roles": {"0": "helper", "1": "main"}, "gpu": [1, 0], "gpu_order": "as_given", "args": []}
+        self.assertEqual(gpu_list(cfg), [1, 0])
+        self.assertTrue(any("edited by hand" in l for l in card_roles_lines(cfg)))
+        self.assertEqual(ordered_gpus({**cfg, "gpu_order": None}, {0: 1.0, 1: 9.0}), [0, 1])       # the legacy rule again
+
+    def test_bad_roles_are_ignored(self):
+        for bad in ({"0": "boss"}, {"0": "main", "1": "main"}, "x", {}):
+            self.assertEqual(gpu_list({"gpu_roles": bad, "gpu": 1, "args": []}), [1])
+
+    def test_legacy_settings_are_untouched(self):
+        self.assertEqual(gpu_list({"gpu": "0,2", "args": []}), [0, 2])
+        self.assertEqual(gpu_list({"args": []}), [])
+        self.assertEqual(card_roles_lines({"gpu": [1, 0], "args": []}), [])
+
+    def test_the_log_line_names_every_role(self):
+        lines = card_roles_lines({"gpu_roles": {"0": "helper", "1": "main", "2": "off"}, "gpu": [0, 1], "args": []})
+        self.assertEqual(lines, ["[strata] card roles: Main GPU 1, Helper GPU 0, Off GPU 2 (numbers as nvidia-smi gives them)"])
+
+    def test_every_card_off(self):
+        cfg = {"gpu_roles": {"0": "off", "1": "off"}, "gpu": [], "gpu_none": True, "args": []}
+        self.assertEqual(gpu_list(cfg), [])
+        env = child_env(cfg)
+        self.assertEqual((env["CUDA_VISIBLE_DEVICES"], env["HIP_VISIBLE_DEVICES"]), ("", ""))
+        with self.assertRaises(SystemExit) as e:
+            card_roles_lines(cfg)                                                   # a CUDA engine cannot start without a card: said plainly
+        self.assertIn("Turn a card on", str(e.exception))
+        cpu = card_roles_lines({**cfg, "backend": "cpu"})
+        self.assertIn("Performance is degraded on CPU only.", cpu[0])
+
+
+if __name__ == "__main__":
+    unittest.main()