mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-10 13:11:31 -10:00
* harden background daemon: stdout integrity, lifecycle, single-instance lock Deep review of internal/daemon (worker pool, launcher, control server, session fan-out, single-instance lock) found and fixes a stdout-integrity bug plus several lifecycle/locking hazards. Each fix has a regression test; the daemon suite passes under -race. Worker stdout integrity - Replace the capped bufio.Scanner (64 KiB) on a worker's stdout with an uncapped bufio.Reader: a legitimately large stream-json line from our own trusted worker is no longer silently dropped (the 1 MiB cap belongs to untrusted network frames, not this local pipe). - Surface a stdout read error as a run failure instead of swallowing it as a clean success, and kill the worker first so it can't block writing to the now-unread pipe and hang Wait. Control-server lifecycle - Track open connections and close them on Shutdown so a single idle/hostile client can no longer wedge drain (and SIGTERM cleanup) forever. - Guard the listener with a mutex and re-check the shutdown signal after bind so a shutdown requested in the bind window is not lost. - Honor shutdown during buffered-history replay too. Session fan-out - A slow subscriber that drops live lines now receives a namespaced gap notice when its channel drains, instead of silently losing output. - Hold the session lock across the (non-blocking) broadcast sends to close a latent send-on-closed-channel panic when a subscriber cancels mid-send. Single-instance lock - Reclaim a stale lock with an atomic rename-with-verify instead of a blind remove, so two instances starting at once cannot both reclaim and end up both holding the lock; a holder that reacquires in the gap is restored. Client + pool - Bound the control handshake with an I/O deadline so a peer that never replies can't wedge Dial/NewClientConn. - Key the active-worker map by a monotonic id rather than PID (the OS reuses PIDs the instant a worker exits). - Group-terminate on context cancel so a stuck worker's children are not orphaned. * daemon: add -race regression test for the Session.Line lock-hold invariant Session.Line holds s.mu across its non-blocking subscriber sends so cancel()/ finish() cannot close a subscriber channel mid-send (send-on-closed-channel panic). That invariant had no concurrency test. Add one that drives Line concurrently with Subscribe/cancel and finish; it panics with a data race if the lock-hold is reverted and passes with it (verified both ways under -race).
341 lines
10 KiB
Go
341 lines
10 KiB
Go
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"sync"
|
|
)
|
|
|
|
// Session/lease/roster model. Mirrors reference-daemon-code-agent-js/
|
|
// session-manager.js (sessionID -> worker + metrics), lease.js (one active
|
|
// request per worker) and roster.js (the live session registry). Because zero's
|
|
// `exec` worker is one-shot, each session is served by exactly one worker via
|
|
// Pool.Run, so "one active request per worker" holds inherently and the pool's
|
|
// bounded slots provide the queue-when-full behavior.
|
|
|
|
// SessionState is a session's lifecycle phase.
|
|
type SessionState string
|
|
|
|
const (
|
|
// SessionQueued: created, waiting for a free pool slot.
|
|
SessionQueued SessionState = "queued"
|
|
// SessionRunning: a worker is leased and producing output.
|
|
SessionRunning SessionState = "running"
|
|
// SessionDone: the worker finished cleanly.
|
|
SessionDone SessionState = "done"
|
|
// SessionFailed: the worker failed permanently / the run errored.
|
|
SessionFailed SessionState = "failed"
|
|
)
|
|
|
|
// ErrSessionExists is returned when starting a session ID already in use.
|
|
var ErrSessionExists = errors.New("daemon: session already exists")
|
|
|
|
// ErrSessionNotFound is returned by Get/Attach for an unknown session ID.
|
|
var ErrSessionNotFound = errors.New("daemon: session not found")
|
|
|
|
const defaultSessionBuffer = 1024
|
|
|
|
// Session is one routed agent run. It buffers recent output (so a late `attach`
|
|
// sees history) and fans live lines out to subscribers. It implements the pool's
|
|
// Sink (Line) and the lease "started" hook (Started).
|
|
type Session struct {
|
|
id string
|
|
cwd string
|
|
|
|
mu sync.Mutex
|
|
state SessionState
|
|
buffer []string
|
|
maxBuffer int
|
|
lines int
|
|
finished bool
|
|
err error
|
|
exitCode int
|
|
subscribers map[int]*subscriber
|
|
nextSub int
|
|
done chan struct{}
|
|
}
|
|
|
|
// subscriber is one live attach: a bounded channel plus a count of live lines
|
|
// dropped because that channel was full. The drop count is flushed to the consumer
|
|
// as a gap notice (see Line) the moment the channel drains, so a lagging client
|
|
// learns its stream is incomplete instead of silently losing output (D8).
|
|
type subscriber struct {
|
|
ch chan string
|
|
dropped int
|
|
}
|
|
|
|
// gapNotice is a stream-json event line emitted to a subscriber that fell behind,
|
|
// reporting how many live lines it missed. It uses a daemon-namespaced type so a
|
|
// stream-json consumer that switches on "type" ignores it rather than mistaking it
|
|
// for a terminal event; the full output remains available via a fresh attach
|
|
// (which replays the buffer).
|
|
func gapNotice(dropped int) string {
|
|
return fmt.Sprintf(`{"type":"daemon_gap","dropped":%d}`, dropped)
|
|
}
|
|
|
|
func newSession(id, cwd string, maxBuffer int) *Session {
|
|
if maxBuffer <= 0 {
|
|
maxBuffer = defaultSessionBuffer
|
|
}
|
|
return &Session{
|
|
id: id,
|
|
cwd: cwd,
|
|
state: SessionQueued,
|
|
maxBuffer: maxBuffer,
|
|
subscribers: map[int]*subscriber{},
|
|
done: make(chan struct{}),
|
|
}
|
|
}
|
|
|
|
// ID returns the session ID.
|
|
func (s *Session) ID() string { return s.id }
|
|
|
|
// Done is closed when the session finishes (success or failure).
|
|
func (s *Session) Done() <-chan struct{} { return s.done }
|
|
|
|
// Err returns the terminal error (nil on clean completion); valid after Done.
|
|
func (s *Session) Err() error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.err
|
|
}
|
|
|
|
// State returns the current lifecycle phase.
|
|
func (s *Session) State() SessionState {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.state
|
|
}
|
|
|
|
// Started marks the session running once the pool leases a worker (lease hook).
|
|
func (s *Session) Started() {
|
|
s.mu.Lock()
|
|
if s.state == SessionQueued {
|
|
s.state = SessionRunning
|
|
}
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// Line records and broadcasts one stream-json output line.
|
|
func (s *Session) Line(line string) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.lines++
|
|
s.buffer = append(s.buffer, line)
|
|
if len(s.buffer) > s.maxBuffer {
|
|
// Drop the oldest line; the ring keeps only the most recent maxBuffer.
|
|
s.buffer = s.buffer[len(s.buffer)-s.maxBuffer:]
|
|
}
|
|
// Hold s.mu across the (non-blocking) sends. The sends never block — every case
|
|
// has a default — so the lock is held only briefly, and holding it stops
|
|
// cancel()/finish() from closing a subscriber channel mid-send, which would
|
|
// panic with "send on closed channel". It also makes each subscriber's dropped
|
|
// counter mutated under the lock.
|
|
for _, sub := range s.subscribers {
|
|
// Non-blocking: a slow/stalled subscriber must not stall the worker pump.
|
|
if sub.dropped > 0 {
|
|
// The consumer fell behind earlier. Before delivering more output, flush a
|
|
// gap notice so it learns its stream is incomplete (D8). If the channel is
|
|
// still full, keep counting and skip this line too.
|
|
select {
|
|
case sub.ch <- gapNotice(sub.dropped):
|
|
sub.dropped = 0
|
|
default:
|
|
sub.dropped++
|
|
continue
|
|
}
|
|
}
|
|
select {
|
|
case sub.ch <- line:
|
|
default:
|
|
// Channel full: count the drop; the gap notice above will report it once
|
|
// the consumer catches up. Full output stays available via a fresh attach.
|
|
sub.dropped++
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Session) finish(exitCode int, err error) {
|
|
s.mu.Lock()
|
|
s.finished = true
|
|
s.exitCode = exitCode
|
|
s.err = err
|
|
if err != nil {
|
|
s.state = SessionFailed
|
|
} else {
|
|
s.state = SessionDone
|
|
}
|
|
subs := s.subscribers
|
|
s.subscribers = map[int]*subscriber{}
|
|
s.mu.Unlock()
|
|
for _, sub := range subs {
|
|
close(sub.ch)
|
|
}
|
|
close(s.done)
|
|
}
|
|
|
|
// Subscribe returns the buffered history plus a channel of subsequent live lines
|
|
// and a cancel func. If the session has already finished, the live channel is
|
|
// returned already closed. Mirrors attach.js.
|
|
func (s *Session) Subscribe() (buffered []string, live <-chan string, cancel func()) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
buffered = append([]string(nil), s.buffer...)
|
|
if s.finished {
|
|
ch := make(chan string)
|
|
close(ch)
|
|
return buffered, ch, func() {}
|
|
}
|
|
id := s.nextSub
|
|
s.nextSub++
|
|
sub := &subscriber{ch: make(chan string, 256)}
|
|
s.subscribers[id] = sub
|
|
cancel = func() {
|
|
s.mu.Lock()
|
|
if c, ok := s.subscribers[id]; ok {
|
|
delete(s.subscribers, id)
|
|
close(c.ch)
|
|
}
|
|
s.mu.Unlock()
|
|
}
|
|
return buffered, sub.ch, cancel
|
|
}
|
|
|
|
func (s *Session) status() SessionStatus {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return SessionStatus{ID: s.id, State: string(s.state), Lines: s.lines}
|
|
}
|
|
|
|
func (s *Session) isFinished() bool {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.finished
|
|
}
|
|
|
|
// defaultMaxSessions bounds the retained session registry so a long-running
|
|
// daemon does not accumulate finished sessions without limit.
|
|
const defaultMaxSessions = 256
|
|
|
|
// SessionManager owns the live session registry and routes each session to a
|
|
// pool worker. Mirrors session-manager.js + roster.js. Finished sessions are
|
|
// retained (so a late `attach` sees history) up to MaxSessions, past which the
|
|
// oldest FINISHED ones are evicted; running/queued sessions are never evicted.
|
|
type SessionManager struct {
|
|
pool *Pool
|
|
maxBuffer int
|
|
maxSessions int
|
|
|
|
mu sync.Mutex
|
|
sessions map[string]*Session
|
|
order []string // creation order, for FIFO eviction of finished sessions
|
|
}
|
|
|
|
// SessionManagerOptions configures a SessionManager.
|
|
type SessionManagerOptions struct {
|
|
Pool *Pool
|
|
MaxBuffer int // per-session output ring size; 0 => default
|
|
// MaxSessions caps the retained session registry; 0 => default. The oldest
|
|
// FINISHED sessions are evicted once the cap is exceeded.
|
|
MaxSessions int
|
|
}
|
|
|
|
// NewSessionManager builds a manager over pool.
|
|
func NewSessionManager(opts SessionManagerOptions) (*SessionManager, error) {
|
|
if opts.Pool == nil {
|
|
return nil, errors.New("daemon: session manager requires a Pool")
|
|
}
|
|
maxSessions := opts.MaxSessions
|
|
if maxSessions <= 0 {
|
|
maxSessions = defaultMaxSessions
|
|
}
|
|
return &SessionManager{
|
|
pool: opts.Pool,
|
|
maxBuffer: opts.MaxBuffer,
|
|
maxSessions: maxSessions,
|
|
sessions: map[string]*Session{},
|
|
}, nil
|
|
}
|
|
|
|
// Start creates a session for spec and dispatches it to the pool. It returns the
|
|
// Session immediately (non-blocking); the run proceeds in the background and the
|
|
// session transitions queued -> running -> done/failed. A duplicate ID is
|
|
// rejected with ErrSessionExists.
|
|
func (m *SessionManager) Start(ctx context.Context, spec WorkerSpec) (*Session, error) {
|
|
m.mu.Lock()
|
|
if _, exists := m.sessions[spec.Session]; exists {
|
|
m.mu.Unlock()
|
|
return nil, ErrSessionExists
|
|
}
|
|
sess := newSession(spec.Session, spec.Cwd, m.maxBuffer)
|
|
m.sessions[spec.Session] = sess
|
|
m.order = append(m.order, spec.Session)
|
|
m.pruneLocked()
|
|
m.mu.Unlock()
|
|
|
|
go func() {
|
|
code, err := m.pool.Run(ctx, spec, sess)
|
|
sess.finish(code, err)
|
|
}()
|
|
return sess, nil
|
|
}
|
|
|
|
// pruneLocked evicts the oldest FINISHED sessions while the registry exceeds
|
|
// maxSessions, so a long-running daemon does not retain finished sessions without
|
|
// bound. Running/queued sessions are never evicted (a transient burst can exceed
|
|
// the cap until those finish). Caller must hold m.mu.
|
|
func (m *SessionManager) pruneLocked() {
|
|
if len(m.sessions) <= m.maxSessions {
|
|
return
|
|
}
|
|
kept := m.order[:0]
|
|
for _, id := range m.order {
|
|
s, ok := m.sessions[id]
|
|
if !ok {
|
|
continue // already removed
|
|
}
|
|
if len(m.sessions) > m.maxSessions && s.isFinished() {
|
|
delete(m.sessions, id)
|
|
continue
|
|
}
|
|
kept = append(kept, id)
|
|
}
|
|
m.order = kept
|
|
}
|
|
|
|
// Get returns a session by ID.
|
|
func (m *SessionManager) Get(id string) (*Session, bool) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
s, ok := m.sessions[id]
|
|
return s, ok
|
|
}
|
|
|
|
// Attach returns the buffered history + live stream for an existing session.
|
|
func (m *SessionManager) Attach(id string) (buffered []string, live <-chan string, cancel func(), err error) {
|
|
s, ok := m.Get(id)
|
|
if !ok {
|
|
return nil, nil, nil, ErrSessionNotFound
|
|
}
|
|
b, l, c := s.Subscribe()
|
|
return b, l, c, nil
|
|
}
|
|
|
|
// Statuses returns a snapshot of all sessions, sorted by ID for stable output.
|
|
func (m *SessionManager) Statuses() []SessionStatus {
|
|
m.mu.Lock()
|
|
sessions := make([]*Session, 0, len(m.sessions))
|
|
for _, s := range m.sessions {
|
|
sessions = append(sessions, s)
|
|
}
|
|
m.mu.Unlock()
|
|
out := make([]SessionStatus, 0, len(sessions))
|
|
for _, s := range sessions {
|
|
out = append(out, s.status())
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
|
return out
|
|
}
|