Verify retained inbound relay GREEN

.github/workflows/tests.yml

@@ -1,31 +1,15 @@
 name: Tests
-
-# Builds, type-checks and runs every test suite on a clean GitHub runner, and records the file
-# list each published package would contain. Each suite is its own job so one failure does not
-# hide the others. No provider credentials or operator configuration are supplied.
-# Package suites deny external networking; scenario runtime has no external route.
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-  workflow_dispatch:
-
+on: workflow_dispatch
 permissions:
   contents: read
-
-concurrency:
-  group: tests-${{ github.event_name }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
-  build-and-package:
+  notice-recovery:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v4
         with:
+          ref: fff440d42d1ba952bdfc513fbe8078d38b32bbf6
           persist-credentials: false
       - uses: actions/setup-node@v4
         with:
@@ -33,110 +17,24 @@ jobs:
           cache: npm
       - run: npm ci
       - run: npm run build
-      - name: Record the published file list of each package
+      - name: Retained inbound relay regression
+        shell: bash
         run: |
-          set -euo pipefail
-          mkdir -p pack-lists
-          for p in cli daemon tui; do
-            (cd packages/$p && npm pack --dry-run --json --ignore-scripts) \
-              | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{for(const f of JSON.parse(s)[0].files)console.log(f.path)})' \
-              | sort > pack-lists/$p.txt
-            echo "$p: $(wc -l < pack-lists/$p.txt) files"
-          done
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pack-lists
-          path: pack-lists/
-
-  typecheck:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: npm
-      - run: npm ci
-      - run: npm run lint
-
-  repo-checks:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: npm
-      - run: npm ci
-      - run: npm run test:repo
-
-  package-tests:
-    if: ${{ github.event_name != 'push' }}
-    # macOS, the platform OpenRig runs on: several daemon and cli tests read macOS ps columns,
-    # file-inode behaviour and release tags.
-    runs-on: macos-latest
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        package: [daemon, cli, tui, ui]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: npm
-      - run: brew install tmux
-      - run: npm ci
-      - name: Build the packages the tests import
-        run: npm run build
-      - name: Run the suite without ambient credentials or external network
-        run: |
-          suite_root=$(mktemp -d "$RUNNER_TEMP/opr-test.XXXXXX")
-          suite_root=$(cd "$suite_root" && pwd -P)
-          mkdir -p "$suite_root/home" "$suite_root/bin"
-          # Same system bytes, nonprivileged mode; no system file is changed.
-          /usr/bin/install -m 0755 /bin/ps "$suite_root/bin/ps"
-          # Only this job's temporary sockets (including tmux) are reachable.
-          env -i HOME="$suite_root/home" PATH="$PATH" TMPDIR="$suite_root" TMUX_TMPDIR="$suite_root" \
-            CI=true TERM=xterm-256color \
-            sandbox-exec -D IPC_ROOT="$suite_root" -p '(version 1)(allow default)(deny network-outbound)(allow network-outbound (remote ip "localhost:*"))(allow network-outbound (subpath (param "IPC_ROOT")))' \
-            /bin/bash -s "${{ matrix.package }}" <<'SUITE'
-          set -euo pipefail
-          node scripts/ci-macos-process-probe.mjs
-          # The production readers use bare ps. Select real system bytes only after
-          # self/owned-child observation and network/IPC confinement are proved.
-          export PATH="$TMPDIR/bin:$PATH"
-          test "$(command -v ps)" = "$TMPDIR/bin/ps"
-          exec npm run test -w "packages/$1"
-          SUITE
-
-  installed-scenario:
-    runs-on: ubuntu-latest
-    timeout-minutes: 35
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: npm
-      - run: npm ci
-      - name: Installed package, daemon restart, and seeded durability failure
-        run: bash scripts/run-pr-scenarios.sh
+          set +e
+          set -uo pipefail
+          mkdir -p proof
+          git rev-parse HEAD HEAD^{tree} > proof/source.txt
+          node --version > proof/node-version.txt
+          npm --workspace @openrig/daemon test -- test/slack-shared-foundation-wire.test.ts test/slack-shared-foundation.test.ts test/slack-subsystem-cutover.test.ts test/slack-thread-recovery.test.ts test/slack-status-backlog.test.ts test/gateway-channel-lifecycle.test.ts --reporter=verbose 2>&1 | tee proof/daemon.log
+          result=${PIPESTATUS[0]}
+          npm run lint 2>&1 | tee proof/typecheck.log
+          lint_result=${PIPESTATUS[0]}
+          git diff --exit-code > proof/source-unchanged.txt
+          printf 'daemon=%s\ntypecheck=%s\n' "$result" "$lint_result" > proof/exits.txt
+          test "$result" -eq 0 && test "$lint_result" -eq 0
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: installed-scenario-evidence
-          path: dist/pr-scenarios/
+          name: slack-forward-green
+          path: proof/
           if-no-files-found: error

docs/reference/slack-app-setup.md

@@ -280,6 +280,7 @@ decisions, and posts its own replies.
   and the machine checks it when it has `OPENRIG_AUTH_BEARER_TOKEN` set.
 - **What the receiving machine trusts:** a relayed event comes from whoever holds that machine's
   daemon token (or, with no token set, anything that can reach its daemon), not from Slack directly, and nothing checks Slack's own signature on it. The
+  credential issued by `rig host pair` is the daemon API bearer, not a relay-only credential; it also authorizes the other daemon write APIs that use that bearer. The
   machine still looks up the sender in its own list of people, but on a relayed event that's no
   proof a person just typed or clicked it in Slack: it trusts the caller for where the event came
   from.
@@ -297,6 +298,11 @@ decisions, and posts its own replies.
   that's down costs one attempt per retry: its other relays wait for the next one. A relay the
   machine keeps refusing is kept and retried with no limit, because dropping it would lose a
   person's message.
+- **Changing forwarding entries is not a drain or cancellation:** relays already retained keep
+  their original host and remain visible in `rig slack status`, even after the last entry is
+  removed. An enabled connector retries them; disabling it pauses retry without deleting them.
+  Let relays finish before removing a forward entry. A zero dead-letter backlog alone does not
+  prove they have drained: it does not count relay HTTP requests still in flight.
 - **Who answered:** every post ends `from <seat> on <machine>`, using the machine's host id (set
   it with `host.name`).
 - **The hub's own channels** are handled on the hub, as before. A channel can be relayed to one

packages/daemon/src/domain/gateway/slack/slack-subsystem.ts

@@ -239,21 +239,22 @@ function stateDir(home: string): string {
  *  retry pass owns exactly these records. When any file cannot be read the count is
  *  `null`, not 0: a number (or a zero) beside an unknown state still reads as "nothing
  *  stuck" to a consumer that only looks at the count, so the state and the reason tell
- *  the truth and the number says it does not know. Absent when inbound is not configured
- *  (no stores), keeping an inert connector silent. */
+ *  the truth and the number says it does not know. An inert connector still reports
+ *  retained relays, but stays silent when there are none. */
 function deadLetterBacklogFields(
   dead: DeadLetterStore<SlackEvent> | undefined,
   deadActions: DeadLetterStore<SlackBlockActions> | undefined,
   deadForwards?: DeadLetterStore<unknown>,
 ): Record<string, unknown> {
-  if (!dead || !deadActions) return {};
   let records = 0;
   let reason: string | undefined;
-  for (const store of [dead, deadActions, ...(deadForwards ? [deadForwards] : [])] as DeadLetterStore<unknown>[]) {
+  for (const store of [dead, deadActions, deadForwards]) {
+    if (!store) continue;
     const result = store.readResult();
     if (result.ok) records += result.entries.length;
     else reason = reason ?? result.reason;
   }
+  if (!dead && !deadActions && records === 0 && reason === undefined) return {};
   if (reason !== undefined) return { deadLetterBacklog: null, deadLetterBacklogState: "unknown", deadLetterBacklogReason: reason };
   return { deadLetterBacklog: records, deadLetterBacklogState: "ok" };
 }
@@ -271,6 +272,9 @@ export function buildSlackGatewayWire(opts: SlackWireOpts): GatewayWire {
 
   const outboundReady = cfg.enabled && bot !== null && cfg.channel !== null;
   const inboundReady = cfg.enabled && app !== null;
+  // Retained relays belong to their original host, even after forwarding configuration changes.
+  const deadForwards = new DeadLetterStore<{ host: string; envelope: SocketEnvelope }>(
+    path.join(stateDir(opts.home), "slack-inbound-forward-deadletter.jsonl"));
 
   if (!outboundReady && !inboundReady) {
     const missing = !cfg.enabled ? "connector disabled (rig slack enable)" : !bot ? "SLACK_BOT_TOKEN unresolved" : "channel unconfigured";
@@ -281,7 +285,7 @@ export function buildSlackGatewayWire(opts: SlackWireOpts): GatewayWire {
       deliver: async () => ({ ok: false, class: "slack-not-configured", detail: missing }),
       log,
     });
-    return { ...inert, status: () => ({ platform: "slack", configurationDigest: channelStateDigest(cfg), outboundReady: false, inboundReady: false, recovery: { state: "unavailable", reason: "inbound-not-configured" }, inbound: { state: "not-configured" } }) };
+    return { ...inert, status: () => ({ platform: "slack", configurationDigest: channelStateDigest(cfg), outboundReady: false, inboundReady: false, recovery: { state: "unavailable", reason: "inbound-not-configured" }, inbound: { state: "not-configured" }, ...deadLetterBacklogFields(undefined, undefined, deadForwards) }) };
   }
 
   const registrySurface: RegistrySurface = opts.registry ?? { loadHumanRegistry, resolveSlackHandle };
@@ -947,12 +951,9 @@ export function buildSlackGatewayWire(opts: SlackWireOpts): GatewayWire {
   // Both files are the retry pass's custody: the event dead-letter
   // (`slack-inbound-deadletter.jsonl`) and the click dead-letter
   // (`slack-inbound-action-deadletter.jsonl`), which retryActionDeadLetters owns.
-  // An inert wire omits the fields rather than inventing a count for a connector
-  // that is not running.
+  // Retained relays are also visible when the wire is inert; its local stores stay unwired.
   let dead: DeadLetterStore<SlackEvent> | undefined;
   let deadActions: DeadLetterStore<SlackBlockActions> | undefined;
-  // The hub's relays that failed, kept with their host (forwarding.ts).
-  let deadForwards: DeadLetterStore<{ host: string; envelope: SocketEnvelope }> | undefined;
   // A machine of a shared app receives its channels' events from the hub (routes/gateway.ts).
   let acceptForwarded: ((env: SocketEnvelope) => Promise<{ status: string; reason?: string }>) | undefined;
   let relayedReceived = 0; // envelopes a hub relayed here since this wire was built
@@ -1037,14 +1038,11 @@ export function buildSlackGatewayWire(opts: SlackWireOpts): GatewayWire {
           lookupBot: (botId: string) => lookupBotApp(bot, botId, opts.fetchImpl),
         },
       } : {}),
-      ...(forwarding ? {
-        forward: {
-          hostFor: (channel: string | undefined) => cfg.forwardChannels!.find((f) => f.channel === channel)?.host ?? null,
-          send: makeEnvelopeForwarder({ fetchImpl: opts.fetchImpl as typeof fetch | undefined, loadRegistry: opts.loadHostRegistry }),
-        },
-        forwardDeadLetter: (deadForwards = new DeadLetterStore<{ host: string; envelope: SocketEnvelope }>(
-          path.join(stateDir(opts.home), "slack-inbound-forward-deadletter.jsonl"))),
-      } : {}),
+      forward: {
+        hostFor: (channel: string | undefined) => forwarding ? cfg.forwardChannels!.find((f) => f.channel === channel)?.host ?? null : null,
+        send: makeEnvelopeForwarder({ fetchImpl: opts.fetchImpl as typeof fetch | undefined, loadRegistry: opts.loadHostRegistry }),
+      },
+      forwardDeadLetter: deadForwards,
       log,
     }));
     if (cfg.botInbox && (!bot || botInboxProblems(cfg).length > 0)) {

packages/daemon/src/routes/gateway.ts

@@ -254,7 +254,10 @@ export function gatewayRoutes(opts: {
         reason: reason || "enable human delivery", before: state(),
         run: async () => {
           let value = { seeded: 0, onlineStatus: `slack connector already ${enabled ? "enabled" : "disabled"}; no change` };
-          if (cfg.enabled === enabled) return { value, after: state(), effect: "no-op" };
+          if (cfg.enabled === enabled) {
+            if (enabled) value.onlineStatus += ". This call did not reload configuration or tokens. To reload, run rig slack disable --reason <reason>, then rig slack enable; environment token overrides still take precedence.";
+            return { value, after: state(), effect: "no-op" };
+          }
           if (enabled) {
             const registry = loadHumanRegistry(home);
             if (!registry.ok) throw new HTTPException(503, { res: c.json({

packages/daemon/test/gateway-channel-lifecycle.test.ts

@@ -114,7 +114,9 @@ describe("human channel lifecycle at the daemon door", () => {
   it("keeps enable no-op and disable available when the human registry is missing", async () => {
     const f = fixture({ registerHuman: false });
     saveConfig({ ...loadConfig(f.home), enabled: true }, f.home);
-    expect((await f.post("enable")).status).toBe(200);
+    const response = await f.post("enable");
+    expect(response.status).toBe(200);
+    expect(await response.json()).toMatchObject({ onlineStatus: expect.stringContaining("did not reload configuration or tokens") });
     expect(f.receipts().at(-1)).toMatchObject({ effect: "no-op" });
     expect(f.restart).not.toHaveBeenCalled();
     expect((await f.post("disable", { reason: "offline maintenance" })).status).toBe(200);

packages/daemon/test/slack-shared-foundation-wire.test.ts

@@ -16,6 +16,8 @@ import type { WsLike } from "../src/domain/gateway/slack/socket-inbound.js";
 import { DEFAULT_CONFIG, saveConfig, type SlackConnectorConfig } from "../src/domain/gateway/slack/config.js";
 import { resolveSlackHandle } from "../src/domain/gateway/human-registry.js";
 import { setSelfHostId } from "../src/domain/hosts/fanout-contract.js";
+import { DeadLetterStore } from "../src/domain/gateway/slack/state-store.js";
+import type { SocketEnvelope } from "../src/domain/gateway/slack/inbound.js";
 
 const human = "human-ada@external";
 const registry = { ok: true as const, entities: [{ entityId: "human-ada", class: "human" as const, displayName: "Ada", address: human, connectorBindings: [{ kind: "slack" as const, connectorRef: "primary", secretsRef: "env:SLACK_BOT_TOKEN", role: "primary" as const, handle: "UADA" }], prefs: { deliveryClass: "A" as const } }] };
@@ -51,12 +53,14 @@ describe("one shared Slack app across two machines", () => {
     stops.push(() => vps.wire?.stop()); vps.wire.startServices?.();
   }
 
-  function wireHub(): void {
+  function wireHub(forwardChannels: SlackConnectorConfig["forwardChannels"] = [{ channel: "C0VPS04", host: "vps04" }, { channel: "C0VPSWS", host: "vps04" }], enabled = true): void {
+    hub.wire?.stop();
+    sockets = [];
     const secrets = join(hub.home, "fake.env");
     writeFileSync(secrets, "SLACK_BOT_TOKEN=xoxb-EXAMPLE-fake\nSLACK_APP_TOKEN=xapp-EXAMPLE-fake\n");
     saveConfig({
-      ...DEFAULT_CONFIG, enabled: true, channel: "C0BUILD", inboundDestination: "orch-advisor@v-build", secretsEnvFile: secrets,
-      forwardChannels: [{ channel: "C0VPS04", host: "vps04" }, { channel: "C0VPSWS", host: "vps04" }],
+      ...DEFAULT_CONFIG, enabled, channel: "C0BUILD", inboundDestination: "orch-advisor@v-build", secretsEnvFile: secrets,
+      forwardChannels,
     }, hub.home);
     hub.wire = buildSlackGatewayWire({
       home: hub.home, queueRepo: hub.repo, registry: { loadHumanRegistry: () => registry, resolveSlackHandle },
@@ -164,4 +168,25 @@ describe("one shared Slack app across two machines", () => {
     expect(inbound(vps)).toEqual([]);
     expect(inbound(hub)).toEqual([]);
   });
+
+  it.each(["message", "click", "one-entry-left", "disabled"])("keeps a retained relay visible after removing its mapping, retrying only while enabled (%s)", async (kind) => {
+    const envelope: SocketEnvelope = kind === "click"
+      ? { envelope_id: "retained-click", type: "interactive", payload: { type: "block_actions", channel: { id: "C0VPS04" }, actions: [] } }
+      : { envelope_id: "retained-message", type: "events_api", payload: { event: { type: "message", user: "UADA", text: "retained reply", ts: "970.1", channel: "C0VPS04" } } };
+    const store = new DeadLetterStore<{ host: string; envelope: SocketEnvelope }>(join(hub.home, "state/slack-inbound-forward-deadletter.jsonl"));
+    store.append({ host: "vps04", envelope }, 1);
+    wireHub(kind === "one-entry-left" ? [{ channel: "C0VPSWS", host: "vps04" }] : [], kind !== "disabled");
+    expect(hub.wire!.status()).toMatchObject({ deadLetterBacklog: 1, deadLetterBacklogState: "ok" });
+    if (kind === "disabled") {
+      expect(hub.wire!.status()).toMatchObject({ inboundReady: false, outboundReady: false });
+      expect(store.readAll()).toHaveLength(1);
+      return;
+    }
+    await vi.waitFor(() => expect(sockets).toHaveLength(1));
+    sockets[0]!.onopen?.();
+    await vi.waitFor(() => expect(store.readAll()).toEqual([]));
+    expect(vps.wire!.status()).toMatchObject({ relayedInbound: { received: 1 } });
+    if (kind !== "click") expect(inbound(vps)).toHaveLength(1);
+    expect(inbound(hub)).toEqual([]);
+  });
 });