launch: the start waits longer for a big model by itself (10 minutes, about 8 s more per GB above 40 GB, at most 30), the server's own engine wait follows, and the runtime accepts start waits up to 1800 s (#3949 #3459). Own fix instead of PR #3817: that one needs people to set STRATA_ENGINE_READY_S; tests

strata_cli/launch/runner.py

@@ -227,6 +227,11 @@ def tune_words(result: dict | None, code: int | None = None) -> tuple[str, str]:
 
 # ----------------------------------------------------------------------------------------------- run config
 
+def runtime_max_wait() -> int:
+    from strata_control import runtime
+    return int(runtime.MAX_START_WAIT)
+
+
 def read_config(path: Path) -> dict:
     return json.loads(Path(path).read_text(encoding="utf-8-sig"))
 
@@ -883,6 +888,17 @@ class Runner:
         except Exception:                                    # noqa: BLE001
             return 0.0
 
+    def start_wait(self) -> int:
+        """How long the start waits for the model to be ready (#3949 #3459): 10 minutes, more for a big model, which a slow disk needs
+        longer to read (about 8 more seconds per GB above 40 GB, at most 30 minutes). Automatic: nobody sets a variable for it."""
+        gb = 0.0
+        try:
+            row = next((r for r in self.catalog.rows() if r.family == self.s.family and r.model == self.s.model), None)
+            gb = float(getattr(row, "download_gb", 0) or 0)
+        except Exception:                                    # noqa: BLE001
+            gb = 0.0
+        return int(min(runtime_max_wait(), max(600, 600 + (gb - 40.0) * 8.0)))
+
     def download_question(self) -> tuple[str, str]:
         """(title, text) of the question before a download starts, naming exactly what is fetched."""
         return self.model_state().question()
@@ -1249,6 +1265,9 @@ class Runner:
         cfg = derive_config(base, self.s, bool_flags=tune_knobs.BOOL_FLAGS,
                             log=str(launch_dir() / f"{self.name}.engine.log"), gpus=cards_for_launch(self.s, self.catalog, base=base))
         self.use_own_dsv41_engine(cfg)
+        wait = self.start_wait()
+        if wait > 600 and not cfg.get("engine_ready_s"):     # the server's own wait for the engine (900 s) must not end before the start's
+            cfg["engine_ready_s"] = wait + 60
         out = launch_dir() / f"{self.name}.json"
         write_private(out, cfg)
         self.config_path = out
@@ -1588,7 +1607,7 @@ class Runner:
 
         def go() -> None:
             try:
-                box["r"] = runtime.start(name, wait=600)
+                box["r"] = runtime.start(name, wait=self.start_wait())
             except Exception as e:                           # noqa: BLE001
                 box["e"] = e
             finally:

strata_control/runtime.py

@@ -515,6 +515,9 @@ def _check_start_candidate(name: str, expected_profile_revision: str | None,
     return plan, digest
 
 
+MAX_START_WAIT = 1800     # seconds a start may wait for the model: a 250 GB DeepSeek set on a slow disk needs more than 10 minutes (#3949)
+
+
 def start(name: str, *, wait: float = 300, shutdown_timeout: float = 30,
           expected_profile_revision: str | None = None,
           expected_source_sha256: str | None = None,
@@ -535,8 +538,8 @@ def start(name: str, *, wait: float = 300, shutdown_timeout: float = 30,
         if token is None:
             raise ProfileError("the process to follow with --die-with-pid is not running")
         die_with = (die_with_pid, token)
-    if not 0 <= wait <= 600 or not 0 <= shutdown_timeout <= 300:
-        raise ProfileError("start wait must be 0..600 and shutdown timeout 0..300 seconds")
+    if not 0 <= wait <= MAX_START_WAIT or not 0 <= shutdown_timeout <= 300:
+        raise ProfileError(f"start wait must be 0..{MAX_START_WAIT} and shutdown timeout 0..300 seconds")
     if expected_profile_revision is not None and (not isinstance(expected_profile_revision, str) or
             not re.fullmatch(r"[0-9a-f]{16}", expected_profile_revision)):
         raise ProfileError("expected profile revision must be 16 lowercase hex characters")
@@ -706,9 +709,9 @@ def restart(name: str, *, timeout: float = 1000, startup_wait: float = 300,
             allow_source_switch: bool = False,
             allow_source_migration: bool = False,
             expected_source_sha256: str | None = None) -> dict:
-    if (not 1 <= timeout <= 1800 or not 1 <= startup_wait <= 600 or
+    if (not 1 <= timeout <= 1800 or not 1 <= startup_wait <= MAX_START_WAIT or
             type(allow_source_switch) is not bool or type(allow_source_migration) is not bool):
-        raise ProfileError("restart wait must be 1..1800 and startup wait 1..600 seconds")
+        raise ProfileError(f"restart wait must be 1..1800 and startup wait 1..{MAX_START_WAIT} seconds")
     if expected_source_sha256 is not None and (not isinstance(expected_source_sha256, str) or
                                                not re.fullmatch(r"[0-9a-f]{64}", expected_source_sha256)):
         raise ProfileError("expected source SHA-256 must be 64 lowercase hex characters")
@@ -1224,7 +1227,7 @@ class Host:
             return {**result, "profile": self.name}
         if command == "restart":
             startup_wait = request.get("startup_wait", 300)
-            if type(startup_wait) not in (int, float) or not 1 <= startup_wait <= 600:
+            if type(startup_wait) not in (int, float) or not 1 <= startup_wait <= MAX_START_WAIT:
                 return {"ok": False, "error": "invalid_startup_wait", "exit_code": 2}
             revision = request.get("expected_profile_revision")
             if revision is not None and (not isinstance(revision, str) or
@@ -1368,7 +1371,7 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--die-with-pid", type=int)
     parser.add_argument("--die-with-token")
     args = parser.parse_args(argv)
-    if not 0 <= args.shutdown_timeout <= 300 or not 1 <= args.startup_wait <= 600:
+    if not 0 <= args.shutdown_timeout <= 300 or not 1 <= args.startup_wait <= MAX_START_WAIT:
         parser.error("shutdown timeout must be 0..300")
     if args.expected_profile_revision is not None and not re.fullmatch(r"[0-9a-f]{16}", args.expected_profile_revision):
         parser.error("expected profile revision must be 16 lowercase hex characters")

tests/terminal/test_flow2_022.py

@@ -357,6 +357,18 @@ class StoredTuning(Base):
         seen, started = self.run_tune(self.runner_for_tune())
         self.assertTrue(started)
 
+    def test_a_big_model_gets_a_longer_start_wait_by_itself(self):                    # #3949 #3459
+        r = self.runner_for_tune()
+        row = type("Row", (), {"family": r.s.family, "model": r.s.model, "download_gb": 20.0})
+        with mock.patch.object(r.catalog, "rows", return_value=[row]):
+            self.assertEqual(r.start_wait(), 600)                                    # a small model keeps ten minutes
+        row.download_gb = 250.0
+        with mock.patch.object(r.catalog, "rows", return_value=[row]):
+            self.assertEqual(r.start_wait(), 1800)                                   # 600 + 210 x 8, capped at 30 minutes
+        row.download_gb = 83.0
+        with mock.patch.object(r.catalog, "rows", return_value=[row]):
+            self.assertEqual(r.start_wait(), 944)
+
     def test_a_stored_result_is_used_and_said(self):
         saved = self.store()
         r = self.runner_for_tune()

tests/terminal/test_runtime.py

@@ -1308,5 +1308,16 @@ class RuntimeTests(unittest.TestCase):
             os.close(exclusive)
 
 
+class StartWaitLimit(unittest.TestCase):
+    def test_a_start_may_wait_up_to_thirty_minutes_for_a_big_model(self):                    # #3949
+        from strata_control import runtime as rt, profiles as pr
+        self.assertEqual(rt.MAX_START_WAIT, 1800)
+        with self.assertRaisesRegex(pr.ProfileError, "start wait must be 0..1800"):
+            rt.start("x", wait=1801)
+        with mock.patch.object(rt, "_name", side_effect=pr.ProfileError("reached")):
+            with self.assertRaisesRegex(pr.ProfileError, "reached"):
+                rt.start("x", wait=1800)
+
+
 if __name__ == "__main__":
     unittest.main()