mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 06:15:58 -10:00
runtime: the managed owner runtime on Windows (loopback TCP + token, job object, byte-range leases)
strata runtime start/status/stop/verify/restart now work on Windows. The owner transport (loopback-only TCP, 256-bit token, state file in a private directory) was already there; the supervisor around it was fcntl/getuid/killpg only. - strata_control/winproc.py: LockFileEx byte-range leases (shared and exclusive, on a byte beyond the data so the file stays readable) and process identity (OpenProcess + GetProcessTimes). Liveness never uses os.kill(pid, 0): on Windows that is TerminateProcess. - runtime.py: state directory %LOCALAPPDATA%\Strata\runtime with a protected one-entry ACL (existing ones must be unshared); owner-only checks through private_files.is_unshared; the state file's address must be a loopback port; the owner host is detached with CREATE_NEW_PROCESS_GROUP|CREATE_NO_WINDOW; the server runs in a kill-on-close job object (serve/winjob.py, terminate() added), so the whole tree ends with the host however the host ends; a graceful stop closes the server's stdin pipe (--owner-fd 0, serve/owner_guard.py turns the EOF into the Ctrl+C drain path); the deadline kill ends the job; the ownership marker records the child's creation time, so a PID that was reused by another process is not mistaken for the recorded child. A PID from a state file is only compared, never signalled. - A connect failure to a lost owner is a typed ProfileError (exit 3) instead of a raw OSError. - `strata serve` and calibration take the activity lease on Windows too. - tests/terminal/test_runtime.py runs on Windows (32 tests were skipped): a .cmd stands in for the fake engine, the signal tests have Windows variants; new tests: a second client attaches from another process, wrong/stale tokens cannot stop the owner, a recorded PID of an unrelated live process is never touched, a reused PID with another creation time is not the recorded child, the liveness check does not end its target, the directory ACL, shared and exclusive leases.
This commit is contained in:
1 parent
66bc889072
commit
d5be478937
7 files changed
+482
-69
No files matched your search
+30
-4
@@ -1,8 +1,15 @@
|
||||
"""Emergency parent-loss guard for a server launched by the local runtime host.
|
||||
"""Parent-loss guard and graceful-stop channel for a server launched by the local runtime host.
|
||||
|
||||
The inherited pipe has exactly one writer, held by the host. EOF therefore
|
||||
means the host exited, including after SIGKILL. The server has its own process
|
||||
group so its engine subprocesses receive the emergency stop as well.
|
||||
means the host exited, including after SIGKILL.
|
||||
|
||||
POSIX: the server has its own process group so its engine subprocesses receive
|
||||
the emergency stop as well. Windows: the host passes the pipe as the server's
|
||||
standard input (``--owner-fd 0``) and ends the server's whole process tree
|
||||
through a kill-on-close job object if it dies; the pipe is also how the host
|
||||
asks the server to drain and exit (it closes its end), because Windows has no
|
||||
SIGTERM to send. EOF is turned into a Ctrl+C in the main thread, which is the
|
||||
path the server already drains and stops cleanly on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -13,7 +20,26 @@ import stat
|
||||
import threading
|
||||
|
||||
|
||||
def start(fd: int) -> None:
|
||||
def _stop_gracefully() -> None:
|
||||
import _thread
|
||||
_thread.interrupt_main()
|
||||
|
||||
|
||||
def start(fd: int, on_loss=None) -> None:
|
||||
if os.name == "nt":
|
||||
if fd < 0 or not stat.S_ISFIFO(os.fstat(fd).st_mode):
|
||||
raise ValueError("--owner-fd must be an inherited pipe")
|
||||
|
||||
def watch_windows() -> None:
|
||||
try:
|
||||
while os.read(fd, 1):
|
||||
pass
|
||||
except OSError:
|
||||
pass
|
||||
(on_loss or _stop_gracefully)()
|
||||
|
||||
threading.Thread(target=watch_windows, name="strata-owner-guard", daemon=True).start()
|
||||
return
|
||||
if os.name != "posix" or fd < 0 or os.getpgrp() != os.getpid():
|
||||
raise ValueError("--owner-fd requires a dedicated POSIX server process group")
|
||||
if not stat.S_ISFIFO(os.fstat(fd).st_mode):
|
||||
|
||||
@@ -45,6 +45,7 @@ if os.name == "nt":
|
||||
_k32.CreateJobObjectW.argtypes = (wintypes.LPVOID, wintypes.LPCWSTR)
|
||||
_k32.SetInformationJobObject.argtypes = (wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD)
|
||||
_k32.AssignProcessToJobObject.argtypes = (wintypes.HANDLE, wintypes.HANDLE)
|
||||
_k32.TerminateJobObject.argtypes = (wintypes.HANDLE, wintypes.UINT)
|
||||
|
||||
def _make_job():
|
||||
job = _k32.CreateJobObjectW(None, None) # no security attributes: the handle is not inherited
|
||||
@@ -80,6 +81,14 @@ else:
|
||||
return False
|
||||
|
||||
|
||||
def terminate() -> bool:
|
||||
"""End every process in the job now (the deadline kill of an owner that must not leave a tree behind). False when
|
||||
there is no job (nothing was contained, or not Windows)."""
|
||||
if os.name != "nt" or not _job:
|
||||
return False
|
||||
return bool(_k32.TerminateJobObject(_job, 1))
|
||||
|
||||
|
||||
def contain(proc) -> bool:
|
||||
"""Put a started subprocess.Popen in the kill-on-close job. True if it is in; False (harmlessly) otherwise."""
|
||||
global _job
|
||||
|
||||
+1
-1
@@ -739,7 +739,7 @@ def _serve(a: argparse.Namespace) -> int:
|
||||
if a.plan:
|
||||
return _success({"plan": plan}, machine=a.output == "json")
|
||||
from serve.server import main as server_main
|
||||
if os.name != "posix":
|
||||
if os.name not in ("posix", "nt"):
|
||||
return _serve_foreground(server_main, args, plan, output=a.output, quiet=a.quiet)
|
||||
from strata_control import runtime
|
||||
try:
|
||||
|
||||
@@ -167,7 +167,7 @@ def _exclusive_preflight(cfg: dict, profile: dict | None = None) -> None:
|
||||
@contextmanager
|
||||
def _managed_exclusive():
|
||||
"""Keep new owner hosts out for the full calibration engine action."""
|
||||
if os.name != "posix":
|
||||
if os.name not in ("posix", "nt"):
|
||||
yield
|
||||
return
|
||||
from strata_control import runtime
|
||||
|
||||
+159
-56
@@ -1,9 +1,16 @@
|
||||
"""Local owner-controlled supervisor for one saved Strata serving profile.
|
||||
|
||||
The supervisor holds the Popen handle for the server it started. A client never
|
||||
sends a signal to a PID found in a ready file. The control socket and token live
|
||||
in an owner-private directory. This first implementation supports POSIX; Windows
|
||||
needs a qualified owner-only pipe/ACL implementation before it may manage work.
|
||||
sends a signal to a PID found in a ready file. The control channel and token live
|
||||
in an owner-private directory.
|
||||
|
||||
POSIX: an owner-private AF_UNIX socket, flock leases, SIGTERM to the server's own
|
||||
process group. Windows: a loopback-only TCP listener authorised by the 256-bit
|
||||
token (strata_control/owner_transport.py), byte-range leases (winproc.py), a
|
||||
protected one-entry ACL on the state directory, and a kill-on-close job object
|
||||
around the server tree. A PID in a state file is never signalled: it is only
|
||||
compared (with its creation time on Windows) to decide whether the recorded
|
||||
child is still there.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -30,16 +37,46 @@ try:
|
||||
except ImportError: # Windows has no qualified owner transport yet.
|
||||
fcntl = None
|
||||
|
||||
from strata_control import owner_transport, profiles, readiness
|
||||
from strata_control import owner_transport, private_files, profiles, readiness, winproc
|
||||
from serve import launch_identity
|
||||
from strata_control.profiles import ProfileError
|
||||
|
||||
|
||||
MAX_MESSAGE = 64 * 1024
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
WINDOWS = os.name == "nt"
|
||||
_STARTED_HOSTS: dict[str, subprocess.Popen] = {}
|
||||
|
||||
|
||||
def _is_private(details: os.stat_result, path: Path) -> bool:
|
||||
"""Owner-only: POSIX owner and no group/other bits; Windows an ACL that shuts out every other account."""
|
||||
if WINDOWS:
|
||||
try:
|
||||
return private_files.is_unshared(path)
|
||||
except OSError:
|
||||
return False
|
||||
return details.st_uid == os.getuid() and not stat.S_IMODE(details.st_mode) & 0o077
|
||||
|
||||
|
||||
def _lock_file(descriptor: int, *, exclusive: bool) -> bool:
|
||||
"""Take a lease without waiting: True when taken, False when another holder has it."""
|
||||
if WINDOWS:
|
||||
return winproc.try_lock(descriptor, exclusive=exclusive)
|
||||
try:
|
||||
fcntl.flock(descriptor, (fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) | fcntl.LOCK_NB)
|
||||
except OSError as exc:
|
||||
if exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN):
|
||||
return False
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
def _unlock_file(descriptor: int) -> None:
|
||||
"""Windows releases a lock only some time after the handle closes: release it now. POSIX closes."""
|
||||
if WINDOWS:
|
||||
winproc.unlock(descriptor)
|
||||
|
||||
|
||||
def _supported() -> None:
|
||||
"""Refuse only when no transport exists at all.
|
||||
|
||||
@@ -63,14 +100,20 @@ def _name(value: str) -> str:
|
||||
def runtime_dir() -> Path:
|
||||
_supported()
|
||||
override = os.environ.get("STRATA_RUNTIME_DIR")
|
||||
base = Path(override).expanduser() if override else \
|
||||
Path(os.environ.get("XDG_STATE_HOME") or Path.home() / ".local" / "state") / "strata" / "runtime"
|
||||
if override:
|
||||
base = Path(override).expanduser()
|
||||
elif WINDOWS:
|
||||
base = Path(os.environ.get("LOCALAPPDATA") or Path.home() / "AppData" / "Local") / "Strata" / "runtime"
|
||||
else:
|
||||
base = Path(os.environ.get("XDG_STATE_HOME") or Path.home() / ".local" / "state") / "strata" / "runtime"
|
||||
if not base.is_absolute():
|
||||
raise ProfileError("STRATA_RUNTIME_DIR must be absolute")
|
||||
existed = base.exists()
|
||||
base.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||
if WINDOWS and not existed:
|
||||
private_files.restrict_to_current_user(base, directory=True) # no mode bits on NTFS: a protected ACL
|
||||
details = base.lstat()
|
||||
if not stat.S_ISDIR(details.st_mode) or details.st_uid != os.getuid() or \
|
||||
stat.S_IMODE(details.st_mode) & 0o077:
|
||||
if not stat.S_ISDIR(details.st_mode) or not _is_private(details, base):
|
||||
raise ProfileError("runtime directory must be owner-only (mode 0700)", 5)
|
||||
return base
|
||||
|
||||
@@ -82,15 +125,10 @@ def acquire_activity_lease(*, shared: bool) -> int:
|
||||
try:
|
||||
descriptor = os.open(directory / ".activity.lock", flags, 0o600)
|
||||
details = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(details.st_mode) or details.st_uid != os.getuid() or \
|
||||
stat.S_IMODE(details.st_mode) & 0o077:
|
||||
if not stat.S_ISREG(details.st_mode) or not _is_private(details, directory / ".activity.lock"):
|
||||
raise ProfileError("runtime activity lock must be owner-only", 5)
|
||||
try:
|
||||
fcntl.flock(descriptor, (fcntl.LOCK_SH if shared else fcntl.LOCK_EX) | fcntl.LOCK_NB)
|
||||
except OSError as exc:
|
||||
if exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN):
|
||||
raise ProfileError("a managed runtime or calibration owns the local engine lease", 6) from None
|
||||
raise
|
||||
if not _lock_file(descriptor, exclusive=not shared):
|
||||
raise ProfileError("a managed runtime or calibration owns the local engine lease", 6)
|
||||
return descriptor
|
||||
except BaseException:
|
||||
if "descriptor" in locals():
|
||||
@@ -128,15 +166,18 @@ def _read_state(name: str) -> dict | None:
|
||||
details = path.lstat()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if not stat.S_ISREG(details.st_mode) or details.st_uid != os.getuid() or \
|
||||
stat.S_IMODE(details.st_mode) & 0o077 or details.st_size > 16 * 1024:
|
||||
if not stat.S_ISREG(details.st_mode) or not _is_private(details, path) or details.st_size > 16 * 1024:
|
||||
raise ProfileError("runtime state file is not private or is invalid", 5)
|
||||
try:
|
||||
state = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, ValueError):
|
||||
raise ProfileError("runtime state file is unreadable", 5) from None
|
||||
# POSIX: the address is the one derived from the profile. Windows: the port is chosen when the owner binds, so
|
||||
# the state file carries it, and the only address a client may be sent to is a loopback port.
|
||||
address_ok = bool(re.fullmatch(r"tcp:127\.0\.0\.1:[1-9][0-9]{0,4}", str(state.get("socket", "")))) \
|
||||
if WINDOWS and isinstance(state, dict) else (isinstance(state, dict) and state.get("socket") == socket_path)
|
||||
if not isinstance(state, dict) or state.get("schema_version") != 1 or state.get("profile") != name or \
|
||||
state.get("socket") != socket_path or not isinstance(state.get("token"), str) or \
|
||||
not address_ok or not isinstance(state.get("token"), str) or \
|
||||
not re.fullmatch(r"[0-9a-f]{64}", state["token"]):
|
||||
raise ProfileError("runtime state file has an unsupported shape", 5)
|
||||
return state
|
||||
@@ -146,7 +187,12 @@ def _child_marker(name: str, token: str) -> Path:
|
||||
return runtime_dir() / f"{name}-{token}.child.json"
|
||||
|
||||
|
||||
def _pid_exists(pid: int) -> bool:
|
||||
def _pid_exists(pid: int, started: int | None = None) -> bool:
|
||||
"""Whether the recorded child is still there. Never signals: on Windows `os.kill(pid, 0)` would END the process,
|
||||
so liveness is a handle query, and `started` (the creation time recorded with the PID) tells the recorded
|
||||
process from a later one that reused the number."""
|
||||
if WINDOWS:
|
||||
return winproc.pid_alive(pid, started)
|
||||
try:
|
||||
os.kill(pid, 0) # Liveness check only; never signal a PID from saved state.
|
||||
except ProcessLookupError:
|
||||
@@ -160,12 +206,9 @@ def _host_lease_active(name: str) -> bool:
|
||||
path, _, _ = _paths(name)
|
||||
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
try:
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as exc:
|
||||
if exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN):
|
||||
return True
|
||||
raise
|
||||
if not _lock_file(descriptor, exclusive=True):
|
||||
return True
|
||||
_unlock_file(descriptor)
|
||||
return False
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
@@ -182,17 +225,12 @@ def _recover_stale(name: str, *, wait: float) -> bool:
|
||||
try:
|
||||
if os.fstat(descriptor).st_ino != state_path.lstat().st_ino:
|
||||
raise ProfileError("runtime state changed during recovery", 6)
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as exc:
|
||||
if exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN):
|
||||
raise ProfileError("this profile already has a live managed runtime", 6) from None
|
||||
raise
|
||||
if not _lock_file(descriptor, exclusive=True):
|
||||
raise ProfileError("this profile already has a live managed runtime", 6)
|
||||
marker = _child_marker(name, state["token"])
|
||||
try:
|
||||
details = marker.lstat()
|
||||
if not stat.S_ISREG(details.st_mode) or details.st_uid != os.getuid() or \
|
||||
stat.S_IMODE(details.st_mode) & 0o077 or details.st_size > 4096:
|
||||
if not stat.S_ISREG(details.st_mode) or not _is_private(details, marker) or details.st_size > 4096:
|
||||
raise ValueError("invalid owner marker")
|
||||
child = json.loads(marker.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
@@ -201,9 +239,10 @@ def _recover_stale(name: str, *, wait: float) -> bool:
|
||||
child.get("phase") != "owned" or type(child.get("pid")) is not int or child["pid"] <= 0:
|
||||
raise ProfileError("lost host child launch is unverified; inspect before recovery", 6)
|
||||
deadline = time.monotonic() + min(45.0, max(5.0, wait))
|
||||
while _pid_exists(child["pid"]) and time.monotonic() < deadline:
|
||||
started = child.get("started") if type(child.get("started")) is int else None
|
||||
while _pid_exists(child["pid"], started) and time.monotonic() < deadline:
|
||||
time.sleep(0.1)
|
||||
if _pid_exists(child["pid"]):
|
||||
if _pid_exists(child["pid"], started):
|
||||
raise ProfileError("lost host child is still present; retry after it exits", 6)
|
||||
if not isinstance(child.get("ready"), str):
|
||||
raise ProfileError("lost host ready path is invalid; inspect before recovery", 6)
|
||||
@@ -213,8 +252,7 @@ def _recover_stale(name: str, *, wait: float) -> bool:
|
||||
raise ProfileError("lost host ready path is invalid; inspect before recovery", 6)
|
||||
try:
|
||||
details = ready.lstat()
|
||||
if not stat.S_ISREG(details.st_mode) or details.st_uid != os.getuid() or \
|
||||
stat.S_IMODE(details.st_mode) & 0o077:
|
||||
if not stat.S_ISREG(details.st_mode) or not _is_private(details, ready):
|
||||
raise ProfileError("lost host ready file is invalid; inspect before recovery", 6)
|
||||
ready.unlink()
|
||||
except FileNotFoundError:
|
||||
@@ -248,15 +286,19 @@ def _recover_stale(name: str, *, wait: float) -> bool:
|
||||
raise ProfileError("too many lost-host config snapshots; inspect before recovery", 6)
|
||||
for snapshot in snapshots:
|
||||
details = snapshot.lstat()
|
||||
if not stat.S_ISREG(details.st_mode) or details.st_uid != os.getuid() or \
|
||||
stat.S_IMODE(details.st_mode) & 0o077:
|
||||
if not stat.S_ISREG(details.st_mode) or not _is_private(details, snapshot):
|
||||
raise ProfileError("lost-host config snapshot is not private; inspect before recovery", 6)
|
||||
snapshot.unlink()
|
||||
marker.unlink()
|
||||
if WINDOWS: # an open file cannot be deleted on Windows: release the lease first
|
||||
_unlock_file(descriptor)
|
||||
os.close(descriptor)
|
||||
descriptor = -1
|
||||
state_path.unlink()
|
||||
return True
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _write_new(path: Path, data: bytes) -> int:
|
||||
@@ -264,7 +306,8 @@ def _write_new(path: Path, data: bytes) -> int:
|
||||
try:
|
||||
# Keep this lease open in the owner host. The kernel releases it even
|
||||
# after SIGKILL, unlike a PID file or a stale Unix socket.
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if not _lock_file(descriptor, exclusive=True):
|
||||
raise OSError(errno.EWOULDBLOCK, "the runtime lease is held")
|
||||
with os.fdopen(os.dup(descriptor), "wb") as stream:
|
||||
stream.write(data)
|
||||
stream.flush()
|
||||
@@ -286,7 +329,10 @@ def _rpc(name: str, command: str, *, timeout: float = 5, params: dict | None = N
|
||||
raw = (json.dumps(request) + "\n").encode("utf-8")
|
||||
# The transport decides the address family: a private POSIX socket, or the loopback listener
|
||||
# on Windows. The token in `raw` is what authorises the request either way.
|
||||
client = owner_transport.connect(state["socket"], timeout=timeout)
|
||||
try:
|
||||
client = owner_transport.connect(state["socket"], timeout=timeout)
|
||||
except OSError: # nothing listens there any more (a lost owner), or it did not answer in time
|
||||
raise ProfileError("runtime control is unavailable; inspect the private host log", 3) from None
|
||||
try:
|
||||
try:
|
||||
client.sendall(raw)
|
||||
@@ -378,9 +424,11 @@ def start(name: str, *, wait: float = 300, shutdown_timeout: float = 30,
|
||||
command += ["--expected-profile-revision", expected_profile_revision]
|
||||
if expected_source_sha256 is not None:
|
||||
command += ["--expected-source-sha256", expected_source_sha256]
|
||||
detach = ({"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW}
|
||||
if WINDOWS else {"start_new_session": True})
|
||||
proc = subprocess.Popen(command,
|
||||
cwd=ROOT, stdin=subprocess.DEVNULL, stdout=stream,
|
||||
stderr=subprocess.STDOUT, start_new_session=True, close_fds=True)
|
||||
stderr=subprocess.STDOUT, close_fds=True, **detach)
|
||||
_STARTED_HOSTS[name] = proc
|
||||
deadline = time.monotonic() + max(5.0, wait)
|
||||
while _read_state(name) is None and proc.poll() is None and time.monotonic() < deadline:
|
||||
@@ -518,6 +566,7 @@ class Host:
|
||||
self.running = True
|
||||
self.state_fd: int | None = None
|
||||
self.owner_write_fd: int | None = None
|
||||
self.owner_write = None # Windows: the server's stdin pipe, whose EOF asks it to drain and stop
|
||||
self.probe_timeout = startup_wait
|
||||
self.probe_deadline = 0.0
|
||||
self.probe_key: tuple[int, str] | None = None
|
||||
@@ -528,6 +577,10 @@ class Host:
|
||||
payload = {"host_token": self.token, "phase": phase}
|
||||
if pid is not None and ready is not None:
|
||||
payload.update(pid=pid, ready=str(ready))
|
||||
if WINDOWS:
|
||||
started = winproc.process_started(pid)
|
||||
if started is not None:
|
||||
payload["started"] = started # tells this process from a later one with the same number
|
||||
profiles._atomic(_child_marker(self.name, self.token), (json.dumps(payload) + "\n").encode())
|
||||
|
||||
def _plan(self) -> tuple[dict, list[str], Path, bytes]:
|
||||
@@ -592,32 +645,47 @@ class Host:
|
||||
log = self.state_path.parent / f"{self.name}.server.log"
|
||||
self._mark_child("launching")
|
||||
descriptor = os.open(log, flags, 0o600)
|
||||
reader, writer = os.pipe()
|
||||
# POSIX: an inherited pipe with one writer (this host). Windows: the server's standard input is that pipe.
|
||||
reader, writer = (None, None) if WINDOWS else os.pipe()
|
||||
server_args = [*args, "--ready-file", str(ready),
|
||||
"--shutdown-timeout", str(self.shutdown_timeout),
|
||||
"--owner-fd", str(reader)]
|
||||
"--owner-fd", "0" if WINDOWS else str(reader)]
|
||||
attestation_key = secrets.token_bytes(32)
|
||||
expected_attestation = launch_identity.attestation(attestation_key, source_bytes, server_args)
|
||||
child_env = os.environ.copy()
|
||||
child_env["STRATA_OWNER_ATTEST_KEY"] = attestation_key.hex()
|
||||
try:
|
||||
with os.fdopen(descriptor, "ab") as stream:
|
||||
self.child = subprocess.Popen([sys.executable, "-m", "serve.server", *server_args],
|
||||
cwd=ROOT, stdin=subprocess.DEVNULL, stdout=stream,
|
||||
stderr=subprocess.STDOUT, close_fds=True,
|
||||
pass_fds=(reader,), start_new_session=True, env=child_env)
|
||||
if WINDOWS:
|
||||
self.child = subprocess.Popen([sys.executable, "-m", "serve.server", *server_args],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=stream,
|
||||
stderr=subprocess.STDOUT, close_fds=True, env=child_env,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW)
|
||||
# the whole server tree ends with this host, however the host ends (a kill-on-close job)
|
||||
from serve import winjob
|
||||
winjob.contain(self.child)
|
||||
else:
|
||||
self.child = subprocess.Popen([sys.executable, "-m", "serve.server", *server_args],
|
||||
cwd=ROOT, stdin=subprocess.DEVNULL, stdout=stream,
|
||||
stderr=subprocess.STDOUT, close_fds=True,
|
||||
pass_fds=(reader,), start_new_session=True, env=child_env)
|
||||
except BaseException:
|
||||
os.close(writer)
|
||||
if writer is not None:
|
||||
os.close(writer)
|
||||
raise
|
||||
finally:
|
||||
os.close(reader)
|
||||
self.owner_write_fd = writer
|
||||
if reader is not None:
|
||||
os.close(reader)
|
||||
if WINDOWS:
|
||||
self.owner_write = self.child.stdin
|
||||
else:
|
||||
self.owner_write_fd = writer
|
||||
try:
|
||||
self._mark_child("owned", pid=self.child.pid, ready=ready)
|
||||
except BaseException:
|
||||
# No ownership record means a future start cannot safely reclaim
|
||||
# this instance. Stop this child immediately instead.
|
||||
os.killpg(self.child.pid, signal.SIGKILL)
|
||||
self._kill_child_tree()
|
||||
self.child.wait(timeout=10)
|
||||
self._close_owner_pipe()
|
||||
raise
|
||||
@@ -702,11 +770,14 @@ class Host:
|
||||
if self.ready_path is not None:
|
||||
self.ready_path.unlink(missing_ok=True)
|
||||
return {"ok": True, "state": "stopped", "exit_code": child.returncode if child else 0}
|
||||
child.send_signal(signal.SIGTERM)
|
||||
if WINDOWS:
|
||||
self._close_owner_pipe() # EOF on the server's stdin: it drains and stops (no SIGTERM here)
|
||||
else:
|
||||
child.send_signal(signal.SIGTERM)
|
||||
try:
|
||||
code = child.wait(timeout=self.shutdown_timeout + 8)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(child.pid, signal.SIGKILL)
|
||||
self._kill_child_tree()
|
||||
child.wait(timeout=10)
|
||||
self._close_owner_pipe()
|
||||
return {"ok": False, "state": "failed", "error": "server_stop_deadline_expired", "exit_code": 8}
|
||||
@@ -720,6 +791,30 @@ class Host:
|
||||
if self.owner_write_fd is not None:
|
||||
os.close(self.owner_write_fd)
|
||||
self.owner_write_fd = None
|
||||
if self.owner_write is not None:
|
||||
try:
|
||||
self.owner_write.close()
|
||||
except OSError:
|
||||
pass
|
||||
self.owner_write = None
|
||||
|
||||
def _kill_child_tree(self) -> None:
|
||||
"""The deadline kill of the server this host started, and everything it started. Only our own child: POSIX
|
||||
signals its process group, Windows ends the job the child was put in (taskkill of our own child's tree if
|
||||
the job could not be made)."""
|
||||
child = self.child
|
||||
if child is None:
|
||||
return
|
||||
if WINDOWS:
|
||||
from serve import winjob
|
||||
if not winjob.terminate():
|
||||
subprocess.run(["taskkill", "/T", "/F", "/PID", str(child.pid)], capture_output=True)
|
||||
try:
|
||||
child.kill()
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
os.killpg(child.pid, signal.SIGKILL)
|
||||
|
||||
def _restart(self, startup_wait: float, expected_profile_revision: str | None = None,
|
||||
allow_source_switch: bool = False,
|
||||
@@ -955,6 +1050,8 @@ class Host:
|
||||
self.running = False
|
||||
signal.signal(signal.SIGTERM, shutdown)
|
||||
signal.signal(signal.SIGINT, shutdown)
|
||||
if hasattr(signal, "SIGBREAK"): # Windows: the signal a console Ctrl+Break / a closing console sends
|
||||
signal.signal(signal.SIGBREAK, shutdown)
|
||||
self.state_fd = _write_new(self.state_path, (json.dumps({"schema_version": 1, "profile": self.name,
|
||||
"socket": bound_address, "token": self.token,
|
||||
"host_pid": os.getpid()}) + "\n").encode())
|
||||
@@ -1006,6 +1103,11 @@ class Host:
|
||||
self._stop_child()
|
||||
self._close_owner_pipe()
|
||||
listener.close()
|
||||
if WINDOWS and self.state_fd is not None:
|
||||
# an open file cannot be deleted on Windows: the lease ends just before the state file does
|
||||
_unlock_file(self.state_fd)
|
||||
os.close(self.state_fd)
|
||||
self.state_fd = None
|
||||
try:
|
||||
current = _read_state(self.name)
|
||||
if current and current.get("token") == self.token:
|
||||
@@ -1024,6 +1126,7 @@ class Host:
|
||||
self._retire_snapshot(snapshot)
|
||||
if self.state_fd is not None:
|
||||
os.close(self.state_fd)
|
||||
_unlock_file(activity_lease)
|
||||
os.close(activity_lease)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Windows file-lock and process-identity primitives for the managed runtime (ctypes; no pywin32).
|
||||
|
||||
POSIX gets these from ``fcntl.flock`` and ``os.kill(pid, 0)``. Neither exists on Windows, and ``os.kill`` there is
|
||||
``TerminateProcess``: calling it with signal 0 would END the process, so a liveness check must never be written that way.
|
||||
|
||||
* ``try_lock`` / ``unlock``: a byte-range lock (``LockFileEx``) on one byte far beyond the data, so the file stays
|
||||
readable by other processes while a lock is held (a lock that covered the data would make reads fail). It supports
|
||||
shared locks, which ``msvcrt.locking`` does not, and is released by the OS when the process ends, however it ends.
|
||||
* ``process_started`` / ``pid_alive``: whether a process exists and when it was created. A PID recorded in a state
|
||||
file is only a hint; the creation time is what tells the process that was recorded from a later one that reused
|
||||
the number.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
WINDOWS = os.name == "nt"
|
||||
|
||||
#: the locked byte. Beyond any data file the runtime writes (they are all far smaller).
|
||||
LOCK_OFFSET = 1 << 16
|
||||
|
||||
if WINDOWS:
|
||||
import ctypes
|
||||
import msvcrt
|
||||
from ctypes import wintypes
|
||||
|
||||
_k32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
|
||||
class _Overlapped(ctypes.Structure):
|
||||
_fields_ = [("Internal", ctypes.c_void_p), ("InternalHigh", ctypes.c_void_p),
|
||||
("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD), ("hEvent", wintypes.HANDLE)]
|
||||
|
||||
_k32.LockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD,
|
||||
ctypes.POINTER(_Overlapped)]
|
||||
_k32.LockFileEx.restype = wintypes.BOOL
|
||||
_k32.UnlockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD,
|
||||
ctypes.POINTER(_Overlapped)]
|
||||
_k32.UnlockFileEx.restype = wintypes.BOOL
|
||||
_k32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
||||
_k32.OpenProcess.restype = wintypes.HANDLE
|
||||
_k32.CloseHandle.argtypes = [wintypes.HANDLE]
|
||||
_k32.GetExitCodeProcess.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)]
|
||||
_k32.GetExitCodeProcess.restype = wintypes.BOOL
|
||||
_k32.GetProcessTimes.argtypes = [wintypes.HANDLE] + [ctypes.POINTER(wintypes.FILETIME)] * 4
|
||||
_k32.GetProcessTimes.restype = wintypes.BOOL
|
||||
|
||||
_LOCKFILE_FAIL_IMMEDIATELY = 0x1
|
||||
_LOCKFILE_EXCLUSIVE_LOCK = 0x2
|
||||
_ERROR_LOCK_VIOLATION = 33
|
||||
_ERROR_IO_PENDING = 997
|
||||
_PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
|
||||
_STILL_ACTIVE = 259
|
||||
_ERROR_ACCESS_DENIED = 5
|
||||
|
||||
def _overlapped() -> _Overlapped:
|
||||
o = _Overlapped()
|
||||
o.Offset = LOCK_OFFSET & 0xFFFFFFFF
|
||||
o.OffsetHigh = LOCK_OFFSET >> 32
|
||||
return o
|
||||
|
||||
def try_lock(fd: int, *, exclusive: bool) -> bool:
|
||||
"""Take the lock without waiting. False when another holder has it."""
|
||||
handle = msvcrt.get_osfhandle(fd)
|
||||
flags = _LOCKFILE_FAIL_IMMEDIATELY | (_LOCKFILE_EXCLUSIVE_LOCK if exclusive else 0)
|
||||
if _k32.LockFileEx(handle, flags, 0, 1, 0, ctypes.byref(_overlapped())):
|
||||
return True
|
||||
code = ctypes.get_last_error()
|
||||
if code in (_ERROR_LOCK_VIOLATION, _ERROR_IO_PENDING):
|
||||
return False
|
||||
raise ctypes.WinError(code, "LockFileEx")
|
||||
|
||||
def unlock(fd: int) -> None:
|
||||
_k32.UnlockFileEx(msvcrt.get_osfhandle(fd), 0, 1, 0, ctypes.byref(_overlapped()))
|
||||
|
||||
def process_started(pid: int) -> int | None:
|
||||
"""The creation time (100 ns ticks since 1601) of a running process, or None when there is no such
|
||||
process, it has exited, or its times cannot be read."""
|
||||
if type(pid) is not int or pid <= 0:
|
||||
return None
|
||||
handle = _k32.OpenProcess(_PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
|
||||
if not handle:
|
||||
return None
|
||||
try:
|
||||
code = wintypes.DWORD()
|
||||
if not _k32.GetExitCodeProcess(handle, ctypes.byref(code)) or code.value != _STILL_ACTIVE:
|
||||
return None
|
||||
created, exited, kernel, user = (wintypes.FILETIME() for _ in range(4))
|
||||
if not _k32.GetProcessTimes(handle, ctypes.byref(created), ctypes.byref(exited),
|
||||
ctypes.byref(kernel), ctypes.byref(user)):
|
||||
return None
|
||||
return (created.dwHighDateTime << 32) | created.dwLowDateTime
|
||||
finally:
|
||||
_k32.CloseHandle(handle)
|
||||
|
||||
def pid_alive(pid: int, started: int | None = None) -> bool:
|
||||
"""Whether `pid` is a running process, and, when `started` is given, the one created at that time."""
|
||||
if type(pid) is not int or pid <= 0:
|
||||
return False
|
||||
now = process_started(pid)
|
||||
if now is None:
|
||||
# OpenProcess can be refused for a protected process that does exist; treat that as present.
|
||||
handle = _k32.OpenProcess(_PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
|
||||
if handle:
|
||||
_k32.CloseHandle(handle)
|
||||
return False # opened, but exited or unreadable: not running
|
||||
return ctypes.get_last_error() == _ERROR_ACCESS_DENIED
|
||||
return started is None or now == started
|
||||
else:
|
||||
def try_lock(fd: int, *, exclusive: bool) -> bool:
|
||||
raise OSError("Windows only")
|
||||
|
||||
def unlock(fd: int) -> None:
|
||||
raise OSError("Windows only")
|
||||
|
||||
def process_started(pid: int) -> int | None:
|
||||
return None
|
||||
|
||||
def pid_alive(pid: int, started: int | None = None) -> bool:
|
||||
raise OSError("Windows only")
|
||||
@@ -21,10 +21,13 @@ from unittest import mock
|
||||
from strata_cli.cli import main
|
||||
from strata_cli import bench
|
||||
from strata_cli.client import Endpoint
|
||||
from strata_control import calibration, journal, management, profiles, recipes, runtime
|
||||
from strata_control import calibration, journal, management, owner_transport, profiles, recipes, runtime
|
||||
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "owner-only Unix socket transport is POSIX-only")
|
||||
WINDOWS = os.name == "nt"
|
||||
|
||||
|
||||
@unittest.skipUnless(os.name in ("posix", "nt"), "no qualified owner transport on this platform")
|
||||
class RuntimeTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(prefix="strata-runtime-test-")
|
||||
@@ -56,6 +59,11 @@ class RuntimeTests(unittest.TestCase):
|
||||
" print('DONE 1 1 1.0 1.0 length 0 0 0', flush=True)\n"
|
||||
" if slot is not None: print(f'BADM {slot} 0', flush=True)\n", encoding="utf-8")
|
||||
engine.chmod(0o700)
|
||||
if WINDOWS:
|
||||
# a script cannot be an engine exe on Windows: a .cmd that runs it stands in for the binary
|
||||
wrapper = self.root / "engine.cmd"
|
||||
wrapper.write_bytes(f'@echo off\r\n"{sys.executable}" "{engine}" %*\r\n'.encode())
|
||||
engine = wrapper
|
||||
from tools.strata_tokenizer import BYTE_TO_UNICODE
|
||||
tokenizer = self.root / "tokenizer"
|
||||
tokenizer.mkdir()
|
||||
@@ -108,7 +116,7 @@ class RuntimeTests(unittest.TestCase):
|
||||
code, invalid = send("true")
|
||||
self.assertEqual((code, invalid["ok"]), (2, False))
|
||||
self.assertIn("live must be true or false", invalid["error"])
|
||||
engine = self.root / "engine.py"
|
||||
engine = self.root / ("engine.cmd" if WINDOWS else "engine.py")
|
||||
engine.unlink()
|
||||
code, missing = send(False)
|
||||
self.assertEqual((code, missing["ok"], missing["report"]["ok"]), (0, True, False))
|
||||
@@ -416,6 +424,7 @@ class RuntimeTests(unittest.TestCase):
|
||||
finally:
|
||||
host.wait(timeout=10)
|
||||
|
||||
@unittest.skipIf(WINDOWS, "Windows console stop needs a platform-specific integration runner (no SIGTERM/SIGINT)")
|
||||
def test_foreground_jsonl_emits_ready_and_stop_without_human_stdout(self):
|
||||
for stop_signal, expected_code, expected_reason, quiet in (
|
||||
(signal.SIGTERM, 0, "sigterm", False),
|
||||
@@ -676,7 +685,11 @@ class RuntimeTests(unittest.TestCase):
|
||||
self.assertEqual(json.loads(self.config.read_text())["port"], edited["port"])
|
||||
snapshot = Path(result["rollback_source_config"])
|
||||
self.assertEqual(snapshot.parent, self.root / "runtime")
|
||||
self.assertEqual(snapshot.stat().st_mode & 0o077, 0)
|
||||
if WINDOWS:
|
||||
from strata_control import private_files
|
||||
self.assertTrue(private_files.is_unshared(snapshot)) # NTFS has no mode bits: the ACL is the promise
|
||||
else:
|
||||
self.assertEqual(snapshot.stat().st_mode & 0o077, 0)
|
||||
self.assertEqual(json.loads(snapshot.read_text())["port"], 0)
|
||||
self.assertEqual(json.loads(snapshot.read_text())["api_key"], secret)
|
||||
self.assertNotIn(secret, json.dumps(result))
|
||||
@@ -970,6 +983,7 @@ class RuntimeTests(unittest.TestCase):
|
||||
code = main(["ops", *args, "--profile", "local", "--output", "json"])
|
||||
return code, json.loads(output.getvalue())
|
||||
|
||||
@unittest.skipIf(WINDOWS, "SIGTERM to a Windows process is TerminateProcess: the hard-kill test below covers it")
|
||||
def test_terminating_owner_host_stops_its_child_and_removes_state(self):
|
||||
started = runtime.start("local", wait=10, shutdown_timeout=2)
|
||||
self.assertTrue(started["ok"])
|
||||
@@ -990,9 +1004,18 @@ class RuntimeTests(unittest.TestCase):
|
||||
orphan = self.root / "runtime" / f"local-{state['token']}-{'a' * 32}.rollback.json"
|
||||
orphan.write_bytes(self.config.read_bytes())
|
||||
orphan.chmod(0o600)
|
||||
os.kill(started["host_pid"], signal.SIGKILL)
|
||||
owned = runtime._STARTED_HOSTS.pop("local")
|
||||
self.assertEqual(owned.wait(timeout=3), -signal.SIGKILL)
|
||||
if WINDOWS:
|
||||
owned.kill() # TerminateProcess: the host cannot clean up after itself
|
||||
owned.wait(timeout=3)
|
||||
# the job object ends the server tree with the host, no matter how the host ended
|
||||
deadline = time.monotonic() + 15
|
||||
while runtime._pid_exists(old_child) and time.monotonic() < deadline:
|
||||
time.sleep(0.1)
|
||||
self.assertFalse(runtime._pid_exists(old_child))
|
||||
else:
|
||||
os.kill(started["host_pid"], signal.SIGKILL)
|
||||
self.assertEqual(owned.wait(timeout=3), -signal.SIGKILL)
|
||||
stale = runtime.status("local")
|
||||
self.assertEqual((stale["state"], stale["error"]), ("stale", "owner_host_lost"))
|
||||
# The next start waits for the inherited-pipe guard to stop the old
|
||||
@@ -1006,7 +1029,8 @@ class RuntimeTests(unittest.TestCase):
|
||||
def test_recovery_refuses_a_still_present_child(self):
|
||||
state_path, socket_path, _ = runtime._paths("local")
|
||||
token = "a" * 64
|
||||
state = {"schema_version": 1, "profile": "local", "socket": str(socket_path),
|
||||
state = {"schema_version": 1, "profile": "local",
|
||||
"socket": "tcp:127.0.0.1:50000" if WINDOWS else str(socket_path),
|
||||
"token": token, "host_pid": 999999}
|
||||
lease = runtime._write_new(state_path, (json.dumps(state) + "\n").encode())
|
||||
os.close(lease)
|
||||
@@ -1019,6 +1043,137 @@ class RuntimeTests(unittest.TestCase):
|
||||
runtime.start("local", wait=0, shutdown_timeout=2)
|
||||
self.assertTrue(state_path.exists())
|
||||
|
||||
# ---- owner transport, attach and identity (POSIX socket and Windows loopback alike)
|
||||
|
||||
def _stale_host(self, child_pid, *, started=None, token="a" * 64):
|
||||
"""A state file whose host is gone, and the ownership marker naming `child_pid` as its server."""
|
||||
state_path, socket_path, _ = runtime._paths("local")
|
||||
state = {"schema_version": 1, "profile": "local",
|
||||
"socket": "tcp:127.0.0.1:50000" if WINDOWS else str(socket_path),
|
||||
"token": token, "host_pid": 999999}
|
||||
lease = runtime._write_new(state_path, (json.dumps(state) + "\n").encode())
|
||||
runtime._unlock_file(lease)
|
||||
os.close(lease)
|
||||
marker = runtime._child_marker("local", token)
|
||||
record = {"host_token": token, "phase": "owned", "pid": child_pid,
|
||||
"ready": str(state_path.parent / "local-fake.ready.json")}
|
||||
if started is not None:
|
||||
record["started"] = started
|
||||
marker.write_text(json.dumps(record), encoding="utf-8")
|
||||
marker.chmod(0o600)
|
||||
return state_path, marker
|
||||
|
||||
def _send(self, state, token, command):
|
||||
client = owner_transport.connect(state["socket"], timeout=5)
|
||||
try:
|
||||
client.sendall((json.dumps({"token": token, "command": command}) + "\n").encode())
|
||||
return json.loads(client.recv(4096).split(b"\n")[0])
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
def test_a_second_client_attaches_from_another_process_and_shares_the_owner(self):
|
||||
started = runtime.start("local", wait=10, shutdown_timeout=2)
|
||||
self.assertTrue(started["ok"])
|
||||
state = runtime._read_state("local")
|
||||
if WINDOWS:
|
||||
self.assertRegex(state["socket"], r"^tcp:127\.0\.0\.1:[1-9][0-9]*$")
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
done = subprocess.run([sys.executable, "strata.py", "runtime", "status", "--profile", "local",
|
||||
"--output", "json"], cwd=root, env=os.environ.copy(), capture_output=True,
|
||||
text=True, timeout=30)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
other = json.loads(done.stdout)
|
||||
self.assertEqual((other["state"], other["owner_id"], other["instance_id"], other["host_pid"]),
|
||||
("ready", started["owner_id"], started["instance_id"], started["host_pid"]))
|
||||
self.assertEqual(runtime.status("local")["owner_id"], started["owner_id"]) # attaching took nothing over
|
||||
self.assertEqual(runtime.stop("local", timeout=12)["ok"], True)
|
||||
self.assertEqual(runtime.status("local")["state"], "stopped")
|
||||
|
||||
def test_a_wrong_or_stale_token_cannot_stop_the_owner(self):
|
||||
started = runtime.start("local", wait=10, shutdown_timeout=2)
|
||||
self.assertTrue(started["ok"])
|
||||
state = runtime._read_state("local")
|
||||
wrong_last = "0" if state["token"][-1] != "0" else "1"
|
||||
for bad in ("0" * 64, state["token"][:-1] + wrong_last, "", "x" * 10):
|
||||
with self.subTest(token=bad[:6]):
|
||||
answer = self._send(state, bad, "stop")
|
||||
self.assertEqual((answer["ok"], answer["error"]), (False, "unauthorized"))
|
||||
self.assertEqual(runtime.status("local")["instance_id"], started["instance_id"])
|
||||
self.assertTrue(self._send(state, state["token"], "status")["ok"]) # the real token is what authorises
|
||||
|
||||
def test_a_recorded_pid_of_an_unrelated_live_process_is_never_signalled(self):
|
||||
victim = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"])
|
||||
try:
|
||||
state_path, marker = self._stale_host(victim.pid)
|
||||
with self.assertRaisesRegex(profiles.ProfileError, "still present"):
|
||||
runtime.start("local", wait=0, shutdown_timeout=2)
|
||||
with self.assertRaises(profiles.ProfileError): # nothing answers on the recorded address
|
||||
runtime.stop("local", timeout=2)
|
||||
self.assertIsNone(victim.poll(), "an unrelated process was stopped")
|
||||
self.assertTrue(state_path.exists() and marker.exists())
|
||||
finally:
|
||||
victim.kill()
|
||||
victim.wait(timeout=10)
|
||||
|
||||
@unittest.skipUnless(WINDOWS, "a creation time is recorded with the PID on Windows only")
|
||||
def test_a_reused_pid_with_another_creation_time_is_not_the_recorded_child(self):
|
||||
victim = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"])
|
||||
try:
|
||||
real = runtime.winproc.process_started(victim.pid)
|
||||
self.assertTrue(runtime._pid_exists(victim.pid, real))
|
||||
self.assertFalse(runtime._pid_exists(victim.pid, real + 12345))
|
||||
self._stale_host(victim.pid, started=real + 12345) # the number was reused by another process
|
||||
started = runtime.start("local", wait=10, shutdown_timeout=2)
|
||||
self.assertTrue(started["ok"]) # recovered: it was not the recorded child
|
||||
self.assertIsNone(victim.poll(), "the process that reused the PID was stopped")
|
||||
finally:
|
||||
victim.kill()
|
||||
victim.wait(timeout=10)
|
||||
|
||||
@unittest.skipUnless(WINDOWS, "os.kill(pid, 0) ends a process on Windows; liveness is a handle query")
|
||||
def test_liveness_check_does_not_end_the_process_it_asks_about(self):
|
||||
victim = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"])
|
||||
try:
|
||||
for _ in range(5):
|
||||
self.assertTrue(runtime._pid_exists(victim.pid))
|
||||
self.assertIsNone(victim.poll())
|
||||
finally:
|
||||
victim.kill()
|
||||
victim.wait(timeout=10)
|
||||
self.assertFalse(runtime._pid_exists(victim.pid))
|
||||
|
||||
@unittest.skipUnless(WINDOWS, "ACLs stand in for mode 0700 on Windows")
|
||||
def test_runtime_directory_is_owner_only_and_a_shared_one_is_refused(self):
|
||||
from strata_control import private_files
|
||||
directory = runtime.runtime_dir()
|
||||
self.assertTrue(private_files.is_private(directory))
|
||||
shared = Path(self.temp.name) / "shared-runtime"
|
||||
shared.mkdir()
|
||||
done = subprocess.run(["icacls", str(shared), "/grant", "*S-1-1-0:(OI)(CI)R"], capture_output=True)
|
||||
if done.returncode != 0:
|
||||
self.skipTest("icacls could not widen the test directory")
|
||||
with mock.patch.dict(os.environ, {"STRATA_RUNTIME_DIR": str(shared)}):
|
||||
with self.assertRaisesRegex(profiles.ProfileError, "owner-only"):
|
||||
runtime.runtime_dir()
|
||||
|
||||
def test_exclusive_lease_waits_for_shared_holders_and_shared_holders_coexist(self):
|
||||
first = runtime.acquire_activity_lease(shared=True)
|
||||
second = runtime.acquire_activity_lease(shared=True)
|
||||
try:
|
||||
with self.assertRaisesRegex(profiles.ProfileError, "engine lease"):
|
||||
runtime.acquire_activity_lease(shared=False)
|
||||
finally:
|
||||
for descriptor in (first, second):
|
||||
runtime._unlock_file(descriptor)
|
||||
os.close(descriptor)
|
||||
exclusive = runtime.acquire_activity_lease(shared=False)
|
||||
try:
|
||||
with self.assertRaisesRegex(profiles.ProfileError, "engine lease"):
|
||||
runtime.acquire_activity_lease(shared=True)
|
||||
finally:
|
||||
runtime._unlock_file(exclusive)
|
||||
os.close(exclusive)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user