Verify retained inbound relay COMPLETE

.github/workflows/tests.yml

@@ -1,31 +1,15 @@
 name: Tests
-
-# Builds, type-checks and runs every test suite on a clean GitHub runner, and records the file
-# list each published package would contain. Each suite is its own job so one failure does not
-# hide the others. No provider credentials or operator configuration are supplied.
-# Package suites deny external networking; scenario runtime has no external route.
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-  workflow_dispatch:
-
+on: workflow_dispatch
 permissions:
   contents: read
-
-concurrency:
-  group: tests-${{ github.event_name }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
-  build-and-package:
+  notice-recovery:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v4
         with:
+          ref: 6c0bfaed09935d549b42e3f32d7f58ada374c39f
           persist-credentials: false
       - uses: actions/setup-node@v4
         with:
@@ -33,110 +17,29 @@ jobs:
           cache: npm
       - run: npm ci
       - run: npm run build
-      - name: Record the published file list of each package
+      - name: Retained inbound relay regression
+        shell: bash
         run: |
-          set -euo pipefail
-          mkdir -p pack-lists
-          for p in cli daemon tui; do
-            (cd packages/$p && npm pack --dry-run --json --ignore-scripts) \
-              | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{for(const f of JSON.parse(s)[0].files)console.log(f.path)})' \
-              | sort > pack-lists/$p.txt
-            echo "$p: $(wc -l < pack-lists/$p.txt) files"
-          done
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pack-lists
-          path: pack-lists/
-
-  typecheck:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: npm
-      - run: npm ci
-      - run: npm run lint
-
-  repo-checks:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: npm
-      - run: npm ci
-      - run: npm run test:repo
-
-  package-tests:
-    if: ${{ github.event_name != 'push' }}
-    # macOS, the platform OpenRig runs on: several daemon and cli tests read macOS ps columns,
-    # file-inode behaviour and release tags.
-    runs-on: macos-latest
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        package: [daemon, cli, tui, ui]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: npm
-      - run: brew install tmux
-      - run: npm ci
-      - name: Build the packages the tests import
-        run: npm run build
-      - name: Run the suite without ambient credentials or external network
-        run: |
-          suite_root=$(mktemp -d "$RUNNER_TEMP/opr-test.XXXXXX")
-          suite_root=$(cd "$suite_root" && pwd -P)
-          mkdir -p "$suite_root/home" "$suite_root/bin"
-          # Same system bytes, nonprivileged mode; no system file is changed.
-          /usr/bin/install -m 0755 /bin/ps "$suite_root/bin/ps"
-          # Only this job's temporary sockets (including tmux) are reachable.
-          env -i HOME="$suite_root/home" PATH="$PATH" TMPDIR="$suite_root" TMUX_TMPDIR="$suite_root" \
-            CI=true TERM=xterm-256color \
-            sandbox-exec -D IPC_ROOT="$suite_root" -p '(version 1)(allow default)(deny network-outbound)(allow network-outbound (remote ip "localhost:*"))(allow network-outbound (subpath (param "IPC_ROOT")))' \
-            /bin/bash -s "${{ matrix.package }}" <<'SUITE'
-          set -euo pipefail
-          node scripts/ci-macos-process-probe.mjs
-          # The production readers use bare ps. Select real system bytes only after
-          # self/owned-child observation and network/IPC confinement are proved.
-          export PATH="$TMPDIR/bin:$PATH"
-          test "$(command -v ps)" = "$TMPDIR/bin/ps"
-          exec npm run test -w "packages/$1"
-          SUITE
-
-  installed-scenario:
-    runs-on: ubuntu-latest
-    timeout-minutes: 35
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: npm
-      - run: npm ci
-      - name: Installed package, daemon restart, and seeded durability failure
-        run: bash scripts/run-pr-scenarios.sh
+          set +e
+          set -uo pipefail
+          mkdir -p proof
+          git rev-parse HEAD HEAD^{tree} > proof/source.txt
+          node --version > proof/node-version.txt
+          npm --workspace @openrig/daemon test -- test/slack-shared-foundation-wire.test.ts test/slack-status-backlog.test.ts --reporter=verbose 2>&1 | tee proof/daemon.log
+          result=${PIPESTATUS[0]}
+          npm --workspace @openrig/cli test -- test/slack-cutover-cli.test.ts --reporter=verbose 2>&1 | tee proof/cli.log
+          cli_result=${PIPESTATUS[0]}
+          npm run lint 2>&1 | tee proof/typecheck.log
+          lint_result=${PIPESTATUS[0]}
+          npm --workspace @openrig/cli test -- test/slack-cutover-cli.test.ts --reporter=verbose 2>&1 | tee proof/cli.log
+          cli_result=${PIPESTATUS[0]}
+          git diff --exit-code > proof/source-unchanged.txt
+          printf 'daemon=%s\ntypecheck=%s\n' "$result" "$lint_result" > proof/exits.txt
+          echo "cli=$cli_result" >> proof/exits.txt
+          test "$result" -eq 0 && test "$cli_result" -eq 0 && test "$lint_result" -eq 0
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: installed-scenario-evidence
-          path: dist/pr-scenarios/
+          name: slack-forward-complete
+          path: proof/
           if-no-files-found: error

docs/reference/slack-app-setup.md

@@ -280,6 +280,7 @@ decisions, and posts its own replies.
   and the machine checks it when it has `OPENRIG_AUTH_BEARER_TOKEN` set.
 - **What the receiving machine trusts:** a relayed event comes from whoever holds that machine's
   daemon token (or, with no token set, anything that can reach its daemon), not from Slack directly, and nothing checks Slack's own signature on it. The
+  credential issued by `rig host pair` is the daemon API bearer, not a relay-only credential; it also authorizes the other daemon write APIs that use that bearer. The
   machine still looks up the sender in its own list of people, but on a relayed event that's no
   proof a person just typed or clicked it in Slack: it trusts the caller for where the event came
   from.
@@ -297,10 +298,18 @@ decisions, and posts its own replies.
   that's down costs one attempt per retry: its other relays wait for the next one. A relay the
   machine keeps refusing is kept and retried with no limit, because dropping it would lose a
   person's message.
+- **Changing forwarding entries is not a drain or cancellation:** relays already retained keep
+  their original host and remain visible in `rig slack status`, even after the last entry is
+  removed. An enabled connector retries them; disabling it pauses retry without deleting them.
+  Let relays finish before removing a forward entry. A zero dead-letter backlog alone does not
+  prove they have drained: it does not count relay HTTP requests still in flight.
 - **Who answered:** every post ends `from <seat> on <machine>`, using the machine's host id (set
   it with `host.name`).
 - **The hub's own channels** are handled on the hub, as before. A channel can be relayed to one
   machine only, and never the hub's own default channel.
+  Events that match neither forwarding nor an existing thread or channel route fall back locally
+  to `inboundDestination`; `rig slack status` names that policy and destination. These lists do not
+  establish exclusive channel ownership: two machines can still both handle a channel.
 - **Missed messages:** each machine recovers the channels it owns: the hub its own channels, and
   every other machine its default channel and its channel map's (the hub never recovers a channel
   it relays). See [Reconnect recovery and status](#reconnect-recovery-and-status).

packages/cli/src/commands/slack.ts

@@ -205,6 +205,7 @@ export function slackCommand(deps: SlackDeps = {}): Command {
         for (const r of readiness) log(`  ${r.ok ? "✓" : "✗"} ${r.label}: ${r.detail}`);
         log(`Daemon: ${observation.state}${observation.reason ? ` (${observation.reason})` : ""}`);
         const connector = observation.connector as { configurationDigest?: string;
+          inboundRouting?: { fallback: string; destination: string };
           deadLetterBacklog?: number | null; deadLetterBacklogState?: string; deadLetterBacklogReason?: string;
           inbound?: InboundDeliveryStatus & { state?: string; generation?: number; lastEventAt?: string };
           recovery?: { state?: string; reason?: string; lastScanAt?: string; acceptedThisProcess?: number; deadLetteredThisProcess?: number;
@@ -215,6 +216,7 @@ export function slackCommand(deps: SlackDeps = {}): Command {
               backoff?: Record<string, number>; incomplete?: Record<string, number> };
             limits?: string[] } } | undefined;
         if (connector) {
+          if (connector.inboundRouting?.fallback === "local") log(`  Routing fallback: local to ${connector.inboundRouting.destination} (after forwarding, thread and channel routes)`);
           log(`  Socket: ${connector.inbound?.state ?? "unknown"}; generation ${connector.inbound?.generation ?? "unknown"}; last event ${connector.inbound?.lastEventAt ?? "unknown"}`);
           for (const line of socketDeliveryLines(connector.inbound)) log(`  ${line}`);
           const recovery = connector.recovery;

packages/cli/test/slack-cutover-cli.test.ts

@@ -132,6 +132,16 @@ describe("S10 CLI cutover — config surfaces survive on the daemon-homed module
     await run(slackCommand(deps), ["status"]);
     expect(logs.join("\n")).toContain("IN-DAEMON");
   });
+
+  it("shows the observed local routing fallback and its destination", async () => {
+    const { deps, logs } = makeDeps({ clientFactory: () => ({ get: async <T>() => ({ status: 200,
+      data: { state: "active", connector: { inboundRouting: { fallback: "local", destination: "ops@hub" } } } as T }) }) });
+    await run(slackCommand(deps), ["status"]);
+    expect(logs.join("\n")).toContain("Routing fallback: local to ops@hub (after forwarding, thread and channel routes)");
+    logs.length = 0;
+    await run(slackCommand(deps), ["status", "--json"]);
+    expect(JSON.parse(logs[0]!).observation.connector.inboundRouting).toEqual({ fallback: "local", destination: "ops@hub" });
+  });
 });
 
 

packages/daemon/src/domain/gateway/slack/slack-subsystem.ts

@@ -239,21 +239,22 @@ function stateDir(home: string): string {
  *  retry pass owns exactly these records. When any file cannot be read the count is
  *  `null`, not 0: a number (or a zero) beside an unknown state still reads as "nothing
  *  stuck" to a consumer that only looks at the count, so the state and the reason tell
- *  the truth and the number says it does not know. Absent when inbound is not configured
- *  (no stores), keeping an inert connector silent. */
+ *  the truth and the number says it does not know. An inert connector still reports
+ *  retained relays, but stays silent when there are none. */
 function deadLetterBacklogFields(
   dead: DeadLetterStore<SlackEvent> | undefined,
   deadActions: DeadLetterStore<SlackBlockActions> | undefined,
   deadForwards?: DeadLetterStore<unknown>,
 ): Record<string, unknown> {
-  if (!dead || !deadActions) return {};
   let records = 0;
   let reason: string | undefined;
-  for (const store of [dead, deadActions, ...(deadForwards ? [deadForwards] : [])] as DeadLetterStore<unknown>[]) {
+  for (const store of [dead, deadActions, deadForwards]) {
+    if (!store) continue;
     const result = store.readResult();
     if (result.ok) records += result.entries.length;
     else reason = reason ?? result.reason;
   }
+  if (!dead && !deadActions && records === 0 && reason === undefined) return {};
   if (reason !== undefined) return { deadLetterBacklog: null, deadLetterBacklogState: "unknown", deadLetterBacklogReason: reason };
   return { deadLetterBacklog: records, deadLetterBacklogState: "ok" };
 }
@@ -271,6 +272,10 @@ export function buildSlackGatewayWire(opts: SlackWireOpts): GatewayWire {
 
   const outboundReady = cfg.enabled && bot !== null && cfg.channel !== null;
   const inboundReady = cfg.enabled && app !== null;
+  const inboundRouting = { fallback: "local", destination: cfg.inboundDestination };
+  // Retained relays belong to their original host, even after forwarding configuration changes.
+  const deadForwards = new DeadLetterStore<{ host: string; envelope: SocketEnvelope }>(
+    path.join(stateDir(opts.home), "slack-inbound-forward-deadletter.jsonl"));
 
   if (!outboundReady && !inboundReady) {
     const missing = !cfg.enabled ? "connector disabled (rig slack enable)" : !bot ? "SLACK_BOT_TOKEN unresolved" : "channel unconfigured";
@@ -281,7 +286,7 @@ export function buildSlackGatewayWire(opts: SlackWireOpts): GatewayWire {
       deliver: async () => ({ ok: false, class: "slack-not-configured", detail: missing }),
       log,
     });
-    return { ...inert, status: () => ({ platform: "slack", configurationDigest: channelStateDigest(cfg), outboundReady: false, inboundReady: false, recovery: { state: "unavailable", reason: "inbound-not-configured" }, inbound: { state: "not-configured" } }) };
+    return { ...inert, status: () => ({ platform: "slack", configurationDigest: channelStateDigest(cfg), outboundReady: false, inboundReady: false, inboundRouting, recovery: { state: "unavailable", reason: "inbound-not-configured" }, inbound: { state: "not-configured" }, ...deadLetterBacklogFields(undefined, undefined, deadForwards) }) };
   }
 
   const registrySurface: RegistrySurface = opts.registry ?? { loadHumanRegistry, resolveSlackHandle };
@@ -947,12 +952,9 @@ export function buildSlackGatewayWire(opts: SlackWireOpts): GatewayWire {
   // Both files are the retry pass's custody: the event dead-letter
   // (`slack-inbound-deadletter.jsonl`) and the click dead-letter
   // (`slack-inbound-action-deadletter.jsonl`), which retryActionDeadLetters owns.
-  // An inert wire omits the fields rather than inventing a count for a connector
-  // that is not running.
+  // Retained relays are also visible when the wire is inert; its local stores stay unwired.
   let dead: DeadLetterStore<SlackEvent> | undefined;
   let deadActions: DeadLetterStore<SlackBlockActions> | undefined;
-  // The hub's relays that failed, kept with their host (forwarding.ts).
-  let deadForwards: DeadLetterStore<{ host: string; envelope: SocketEnvelope }> | undefined;
   // A machine of a shared app receives its channels' events from the hub (routes/gateway.ts).
   let acceptForwarded: ((env: SocketEnvelope) => Promise<{ status: string; reason?: string }>) | undefined;
   let relayedReceived = 0; // envelopes a hub relayed here since this wire was built
@@ -1037,14 +1039,11 @@ export function buildSlackGatewayWire(opts: SlackWireOpts): GatewayWire {
           lookupBot: (botId: string) => lookupBotApp(bot, botId, opts.fetchImpl),
         },
       } : {}),
-      ...(forwarding ? {
-        forward: {
-          hostFor: (channel: string | undefined) => cfg.forwardChannels!.find((f) => f.channel === channel)?.host ?? null,
-          send: makeEnvelopeForwarder({ fetchImpl: opts.fetchImpl as typeof fetch | undefined, loadRegistry: opts.loadHostRegistry }),
-        },
-        forwardDeadLetter: (deadForwards = new DeadLetterStore<{ host: string; envelope: SocketEnvelope }>(
-          path.join(stateDir(opts.home), "slack-inbound-forward-deadletter.jsonl"))),
-      } : {}),
+      forward: {
+        hostFor: (channel: string | undefined) => forwarding ? cfg.forwardChannels!.find((f) => f.channel === channel)?.host ?? null : null,
+        send: makeEnvelopeForwarder({ fetchImpl: opts.fetchImpl as typeof fetch | undefined, loadRegistry: opts.loadHostRegistry }),
+      },
+      forwardDeadLetter: deadForwards,
       log,
     }));
     if (cfg.botInbox && (!bot || botInboxProblems(cfg).length > 0)) {
@@ -1118,6 +1117,7 @@ export function buildSlackGatewayWire(opts: SlackWireOpts): GatewayWire {
       platform: "slack", configurationDigest: channelStateDigest(cfg),
       outboundReady,
       inboundReady,
+      inboundRouting,
       recovery: recovery?.status() ?? { state: "unavailable", reason: "inbound-not-configured" },
       inbound: inboundHandle?.status() ?? { state: inboundReady ? "not-started" : "not-configured", generation: 0, reconnects: 0 },
       // The retry pass's durable custody, counted from both dead-letter files:

packages/daemon/src/routes/gateway.ts

@@ -254,7 +254,10 @@ export function gatewayRoutes(opts: {
         reason: reason || "enable human delivery", before: state(),
         run: async () => {
           let value = { seeded: 0, onlineStatus: `slack connector already ${enabled ? "enabled" : "disabled"}; no change` };
-          if (cfg.enabled === enabled) return { value, after: state(), effect: "no-op" };
+          if (cfg.enabled === enabled) {
+            if (enabled) value.onlineStatus += ". This call did not reload configuration or tokens. To reload, run rig slack disable --reason <reason>, then rig slack enable; environment token overrides still take precedence.";
+            return { value, after: state(), effect: "no-op" };
+          }
           if (enabled) {
             const registry = loadHumanRegistry(home);
             if (!registry.ok) throw new HTTPException(503, { res: c.json({

packages/daemon/test/gateway-channel-lifecycle.test.ts

@@ -114,7 +114,9 @@ describe("human channel lifecycle at the daemon door", () => {
   it("keeps enable no-op and disable available when the human registry is missing", async () => {
     const f = fixture({ registerHuman: false });
     saveConfig({ ...loadConfig(f.home), enabled: true }, f.home);
-    expect((await f.post("enable")).status).toBe(200);
+    const response = await f.post("enable");
+    expect(response.status).toBe(200);
+    expect(await response.json()).toMatchObject({ onlineStatus: expect.stringContaining("did not reload configuration or tokens") });
     expect(f.receipts().at(-1)).toMatchObject({ effect: "no-op" });
     expect(f.restart).not.toHaveBeenCalled();
     expect((await f.post("disable", { reason: "offline maintenance" })).status).toBe(200);

packages/daemon/test/slack-shared-foundation-wire.test.ts

@@ -16,6 +16,8 @@ import type { WsLike } from "../src/domain/gateway/slack/socket-inbound.js";
 import { DEFAULT_CONFIG, saveConfig, type SlackConnectorConfig } from "../src/domain/gateway/slack/config.js";
 import { resolveSlackHandle } from "../src/domain/gateway/human-registry.js";
 import { setSelfHostId } from "../src/domain/hosts/fanout-contract.js";
+import { DeadLetterStore } from "../src/domain/gateway/slack/state-store.js";
+import type { SocketEnvelope } from "../src/domain/gateway/slack/inbound.js";
 
 const human = "human-ada@external";
 const registry = { ok: true as const, entities: [{ entityId: "human-ada", class: "human" as const, displayName: "Ada", address: human, connectorBindings: [{ kind: "slack" as const, connectorRef: "primary", secretsRef: "env:SLACK_BOT_TOKEN", role: "primary" as const, handle: "UADA" }], prefs: { deliveryClass: "A" as const } }] };
@@ -51,12 +53,14 @@ describe("one shared Slack app across two machines", () => {
     stops.push(() => vps.wire?.stop()); vps.wire.startServices?.();
   }
 
-  function wireHub(): void {
+  function wireHub(forwardChannels: SlackConnectorConfig["forwardChannels"] = [{ channel: "C0VPS04", host: "vps04" }, { channel: "C0VPSWS", host: "vps04" }], enabled = true): void {
+    hub.wire?.stop();
+    sockets = [];
     const secrets = join(hub.home, "fake.env");
     writeFileSync(secrets, "SLACK_BOT_TOKEN=xoxb-EXAMPLE-fake\nSLACK_APP_TOKEN=xapp-EXAMPLE-fake\n");
     saveConfig({
-      ...DEFAULT_CONFIG, enabled: true, channel: "C0BUILD", inboundDestination: "orch-advisor@v-build", secretsEnvFile: secrets,
-      forwardChannels: [{ channel: "C0VPS04", host: "vps04" }, { channel: "C0VPSWS", host: "vps04" }],
+      ...DEFAULT_CONFIG, enabled, channel: "C0BUILD", inboundDestination: "orch-advisor@v-build", secretsEnvFile: secrets,
+      forwardChannels,
     }, hub.home);
     hub.wire = buildSlackGatewayWire({
       home: hub.home, queueRepo: hub.repo, registry: { loadHumanRegistry: () => registry, resolveSlackHandle },
@@ -150,6 +154,14 @@ describe("one shared Slack app across two machines", () => {
     expect(await answer(hub, "orch-advisor@v-build", row.qitemId, "Seen")).toMatchObject({ channel: "C0BUILD", thread_ts: "730.1" });
   });
 
+  it("reports the local fallback and still routes an unmapped channel to that destination", async () => {
+    expect(hub.wire!.status()).toMatchObject({ inboundRouting: { fallback: "local", destination: "orch-advisor@v-build" } });
+    await say("C0UNMAPPED", "735.1");
+    await vi.waitFor(() => expect(inbound(hub)).toHaveLength(1));
+    expect(inbound(hub)[0]!.destinationSession).toBe("orch-advisor@v-build");
+    expect(inbound(vps)).toEqual([]);
+  });
+
   it("a machine without its own connection reports that it can take relayed events, and how many have arrived", async () => {
     expect(vps.wire!.status()).toMatchObject({ inboundReady: false, relayedInbound: { accepting: true, received: 0 } });
     expect(vps.wire!.status()).not.toHaveProperty("inboundRelayed");
@@ -164,4 +176,25 @@ describe("one shared Slack app across two machines", () => {
     expect(inbound(vps)).toEqual([]);
     expect(inbound(hub)).toEqual([]);
   });
+
+  it.each(["message", "click", "one-entry-left", "disabled"])("keeps a retained relay visible after removing its mapping, retrying only while enabled (%s)", async (kind) => {
+    const envelope: SocketEnvelope = kind === "click"
+      ? { envelope_id: "retained-click", type: "interactive", payload: { type: "block_actions", channel: { id: "C0VPS04" }, actions: [] } }
+      : { envelope_id: "retained-message", type: "events_api", payload: { event: { type: "message", user: "UADA", text: "retained reply", ts: "970.1", channel: "C0VPS04" } } };
+    const store = new DeadLetterStore<{ host: string; envelope: SocketEnvelope }>(join(hub.home, "state/slack-inbound-forward-deadletter.jsonl"));
+    store.append({ host: "vps04", envelope }, 1);
+    wireHub(kind === "one-entry-left" ? [{ channel: "C0VPSWS", host: "vps04" }] : [], kind !== "disabled");
+    expect(hub.wire!.status()).toMatchObject({ deadLetterBacklog: 1, deadLetterBacklogState: "ok" });
+    if (kind === "disabled") {
+      expect(hub.wire!.status()).toMatchObject({ inboundReady: false, outboundReady: false });
+      expect(store.readAll()).toHaveLength(1);
+      return;
+    }
+    await vi.waitFor(() => expect(sockets).toHaveLength(1));
+    sockets[0]!.onopen?.();
+    await vi.waitFor(() => expect(store.readAll()).toEqual([]));
+    expect(vps.wire!.status()).toMatchObject({ relayedInbound: { received: 1 } });
+    if (kind !== "click") expect(inbound(vps)).toHaveLength(1);
+    expect(inbound(hub)).toEqual([]);
+  });
 });