Merge remote-tracking branch 'origin/main' into jas-batch

Invocations that come due within a few milliseconds go out as one
/invoke-batch request (up to events.invokeBatchSize, default 60). The
dispatcher runs a few at a time and streams a started line and an answer line
per call, so a slow handler delays no other call's answer. A call's timeout
runs from its started line, so time queued behind other calls is never the
handler's; queue time comes out of the connection-wait share instead, and a
call still queued past it is never started and fails as Puter's.

A lone call still uses /invoke, and a dispatcher without the batch route gets
single calls for five minutes before the batch route is tried again.

.github/workflows/ai-provider-integration-tests.yaml

@@ -47,3 +47,13 @@ jobs:
                   PUTER_TEST_AI_MOONSHOT_API_KEY: ${{ secrets.PUTER_TEST_AI_MOONSHOT_API_KEY }}
                   PUTER_TEST_AI_ZAI_API_KEY: ${{ secrets.PUTER_TEST_AI_ZAI_API_KEY }}
                   PUTER_TEST_AI_ELEVENLABS_API_KEY: ${{ secrets.PUTER_TEST_AI_ELEVENLABS_API_KEY }}
+
+            # Realtime conversations through the SDK against live OpenAI. Builds
+            # the SDK bundle these tests load itself, so it runs even when the
+            # step above fails; the tests skip themselves without the key.
+            - name: Run live realtime conversation tests
+              if: ${{ !cancelled() }}
+              run: npm run setupExtensions && (cd src/puter-js && npm run build) && npx vitest run --config src/puter-js/tests/api/vitest.config.ts src/puter-js/tests/api/realtime/convoLive.test.ts
+              env:
+                  CI: 'true'
+                  PUTER_TEST_AI_OPENAI_API_KEY: ${{ secrets.PUTER_TEST_AI_OPENAI_API_KEY }}

.github/workflows/puterjs-tests.yaml

@@ -118,9 +118,13 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # The base branch lives in this repo; for a fork PR, the head
+          # repo's copy of it can be arbitrarily stale.
           - ref: ${{ github.base_ref }}
+            repository: ${{ github.repository }}
             artifact: base
           - ref: ${{ github.head_ref }}
+            repository: ${{ github.event.pull_request.head.repo.full_name }}
             artifact: pr
 
     steps:
@@ -129,7 +133,7 @@ jobs:
         uses: actions/checkout@v7
         with:
           ref: ${{ matrix.ref }}
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          repository: ${{ matrix.repository }}
 
       - name: Set up Node.js
         if: needs.changes.outputs.puterjs == 'true'

AGENTS.md

@@ -201,6 +201,6 @@ Recipes are tested, copy-pasteable patterns for a task, one layer above the per-
 
 - **Reuse existing UI primitives before writing new ones.** Windows and dialogs are `UIWindow*` functions in [src/gui/src/UI/](src/gui/src/UI/); generic pieces already exist (`UIAlert`, `UIPrompt`, `UIContextMenu`, `UINotification`, `UIPopover`, widgets in [UI/Components/](src/gui/src/UI/Components/)). A new window should read like its neighbors: an async function taking an options object, composing an HTML string, wiring behavior with jQuery, delegating to `UIWindow(...)`.
 - **Match the visual language.** Use existing CSS classes (`button`, `button-primary`, window chrome, form styles) and copy the layout patterns of neighboring windows; new styles go in [src/gui/src/css/](src/gui/src/css/) following existing conventions. Verify anything positional (menus, overlays, z-index) on both desktop and mobile viewports.
-- **i18n every user-facing string.** No hardcoded UI text — use `i18n('key')` and add the key to [src/gui/src/i18n/translations/en.js](src/gui/src/i18n/translations/en.js). Run `npm run check-translations` before opening the PR.
+- **i18n every user-facing string.** No hardcoded UI text — use `i18n('key')` and add the key to [src/gui/src/i18n/translations/en.js](src/gui/src/i18n/translations/en.js).
 - **Shared logic goes in helpers/services.** Reusable non-UI logic belongs in [src/gui/src/helpers/](src/gui/src/helpers/) or [src/gui/src/services/](src/gui/src/services/), not copy-pasted between windows.
 - **Tests.** Vitest is wired for the GUI (`src/**/*.test.js`; see [appOrder.test.js](src/gui/src/UI/Dashboard/appOrder.test.js) for the shape). Extract pure logic into functions and test those. Desktop behavior driven through puter.js (`puter.ui.*`) is covered by the Playwright harness in [src/puter-js/tests/e2e/](src/puter-js/tests/e2e/) — add a spec there when you change how the desktop renders SDK-driven UI.

Dockerfile

@@ -73,6 +73,12 @@ COPY --from=build --chown=node:node /opt/puter .
 RUN mkdir -p /etc/puter /var/puter && \
     chown -R node:node /etc/puter /var/puter
 
+# Fix outdated HOME_URL in /etc/os-release (see issue #4329)
+# The base image may contain a reference to the old phoenix repository.
+RUN if [ -f /etc/os-release ]; then \
+      sed -i 's|HOME_URL="https://github.com/HeyPuter/phoenix"|HOME_URL="https://github.com/HeyPuter/puter"|' /etc/os-release; \
+    fi
+
 # Self-hosters mount their override at this exact path. The v2 loader
 # deep-merges it over config.default.json (see backend/index.ts).
 ENV PUTER_CONFIG_PATH=/etc/puter/config.json
@@ -87,4 +93,4 @@ USER node
 HEALTHCHECK --interval=30s --timeout=3s --start-period=30s --retries=3 \
     CMD wget --no-verbose --tries=1 --spider http://puter.localhost:4100/test || exit 1
 
-CMD ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/index.js"]
+CMD ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/index.js"]

doc/pagination.md

@@ -59,6 +59,10 @@ store:
   row — and the query seeks past it (`WHERE (col, id) > (?, ?) ORDER BY col,
   id`). Any SQL list gaining a cursor must have a deterministic `ORDER BY`
   ending in a unique tiebreaker (`id`).
+- The notification catch-up (`GET /events/fetch`) holds two keysets in one
+  sealed cursor: the last `id` it returned, and the last `(replaced_at, id)`
+  among rows up to that id. Each is its own indexed read, merged into one
+  page, so a row replaced after a client read it comes back from that cursor.
 
 Cursors that carry a sort also pin it: a request that passes a cursor plus a
 conflicting sort is rejected with 400.

doc/push-client.md

@@ -18,6 +18,17 @@ Core subscribes the browser to a push service only when the server also publishe
 
 When the browser replaces the subscription (`pushsubscriptionchange`), `sw.js` posts `{ type: 'puter-push-subscription-changed' }` to every open window on Puter's origin. The worker has no token or session, so the page's push client registers the new subscription.
 
+## Receipts
+
+Before a notification is pushed, the server asks the open clients whether they received it live. It sends `notif.receipt` with `{ uid }` on the user's sockets for the app that sent it, or on the desktop's sockets for Puter's own notifications. The SDK remembers the uids its page received through `onNotification()`, `onPush()` or the desktop's feed, and answers once its page has that notification, or with `{ received: false }` when it never gets it. The server waits up to 5 seconds (`notificationAckTimeoutMs`).
+
+An answer carries `visibilityState`, `focused` and `showsOwn`, the last being whether the page shows the notification itself. A website page shows it itself when it is hidden or unfocused, has an `onPush()` handler and notification permission, and no other tab of the same site is focused. Inside the desktop, the desktop shows it, and an app's `onPush()` result is never shown as a browser notification.
+
+- An answer from a page that is visible and focused, or that shows the notification itself, stops the push.
+- Otherwise the notification is pushed. A website's pushes go only to the browsers that subscribed through its own worker, never to Puter's `/sw.js`. An app inside the desktop pushes through `/sw.js`, which shows the app's name and icon.
+- A hidden desktop tab shows no banner, so answers only from hidden desktop tabs don't stop a push of Puter's own notifications.
+- A client that can't answer, such as an older puter.js, holds the push back while it is connected.
+
 ## Registering
 
 The registry is in [src/gui/src/helpers/pushClient.js](../src/gui/src/helpers/pushClient.js). A service script registers its client like this:
@@ -57,8 +68,8 @@ interface PushClient {
 }
 ```
 
-- `enable()` may ask for notification permission, so core calls it from a click: Turn on in the Notifications tab, Allow on a permission prompt that includes `notifications:push` while this browser is off, and the "Turn on notifications" button of the `/action/enable-push` popup and the matching desktop dialog (`UIEnablePushDialog`). Core also calls it without a click, from `helpers/pushSync.js`, when `Notification.permission` is already `granted`, `status()` reports the browser as not registered, the user has not turned push off in this browser, and an app holds `notifications:push`; the browser does not prompt then. It subscribes with the `/sw.js` registration, registers the device with `POST /push/devices` (`platform: 'web'`), and resolves the device `uid` the server returned. It rejects when permission is refused or the device couldn't be registered.
-- `disable()` unsubscribes and removes this browser's device with `DELETE /push/devices/:uid`. The tab calls it for Turn off, and for Remove on the device marked as this browser; Remove on any other device is a plain `DELETE`.
+- `enable()` may ask for notification permission, so core calls it from a click: Turn on in the Notifications tab, Allow on a permission prompt that includes `notifications:push` while this browser is off, and the "Turn on notifications" button of the `/action/enable-push` popup and the matching desktop dialog (`UIEnablePushDialog`). Core also calls it without a click, from `helpers/pushSync.js`, when `Notification.permission` is already `granted`, `status()` reports the browser as not registered, the user has not turned push off in this browser, and an app holds `notifications:push`; the browser does not prompt then. The built-in client runs `puter.notifications.enable()`, which subscribes with the `/sw.js` registration and registers the browser through the Web Push flow above (`platform: 'webpush'`). It then resolves the `uid` of this browser's `webpush` device from `listDevices()`. It rejects when permission is refused or the browser wasn't registered.
+- `disable()` runs `puter.notifications.disable()`, which sends `DELETE /push/web/subscriptions { endpoint }` and ends the browser's subscription. The tab calls it for Turn off, and for Remove on the device marked as this browser. Remove on any other device is `DELETE /push/devices/:uid`.
 - `status()` reports the state without prompting. `permission` is `Notification.permission`. `registered` is true when this browser has a device registered for the signed-in user, and `deviceUid` is that device's `uid`.
 
 After `enable()` resolves, the tab calls `POST /push/devices/:uid/test` and asks the user whether a notification showed up.

doc/self-hosting.md

@@ -12,7 +12,17 @@ Generates secrets, writes `.env` + `puter/config/config.json`, downloads `docker
 
 ## Requirements
 
-- **Docker** with the `compose` plugin.
+- **Docker** with the `compose` plugin. This is the one thing you have to install
+  yourself — the installer checks for it and stops with an explanation rather
+  than installing it for you.
+  - **macOS / Windows:** [Docker Desktop](https://docs.docker.com/get-docker/).
+    Both ship the `compose` plugin.
+  - **Linux:** [Docker Engine](https://docs.docker.com/engine/install/) plus the
+    [`docker-compose-plugin`](https://docs.docker.com/compose/install/) package.
+  - Confirm before running anything else: `docker compose version` should print a
+    version, not `command not found`.
+  - If Docker is installed but the shell can't see it, make sure your user is in
+    the `docker` group (Linux) or that Docker Desktop is actually running.
 - A **domain** with DNS access — you need a wildcard record (`*.your-domain.com` → server IP). Puter routes by subdomain (`api.<domain>`, `site.<domain>`, `app.<domain>`).
 - Optional: **TLS certs** (or `certbot` to grab them — see Step 3).
 
@@ -657,6 +667,14 @@ Migrations re-apply idempotently across pulls. Volumes are preserved.
 
 ## Troubleshooting
 
+**`missing required command: docker` — the installer stops immediately.**
+Docker isn't on your `PATH` yet. Puter self-hosting is a Docker Compose stack, so
+nothing can run without it. Install [Docker Desktop](https://docs.docker.com/get-docker/)
+on macOS/Windows or [Docker Engine](https://docs.docker.com/engine/install/) plus
+`docker-compose-plugin` on Linux, then confirm `docker compose version` works in
+the same shell and re-run the installer. On Linux, a fresh Docker install usually
+needs a new login session before your user picks up the `docker` group.
+
 **Site loads but I get a 502 / "Bad Gateway" from Caddy.**
 The puter container failed to come up. `docker compose logs puter` will tell you which dependency rejected it (most often DB password mismatch between `.env` and `config.json`).
 

install.sh

@@ -53,9 +53,29 @@ need() {
     command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
 }
 
+check_docker() {
+    if command -v docker >/dev/null 2>&1; then
+        return 0
+    fi
+    echo >&2 ""
+    echo >&2 "[puter-install] missing required command: docker"
+    echo >&2 ""
+    echo >&2 "Docker is required to run Puter self-hosted."
+    echo >&2 ""
+    echo >&2 "Installation instructions:"
+    echo >&2 "  - macOS (Apple Silicon & Intel): https://docs.docker.com/desktop/setup/install/mac-install/"
+    echo >&2 "    (Docker Desktop is recommended. Alternatives: OrbStack, Colima)"
+    echo >&2 "  - Linux: https://docs.docker.com/engine/install/"
+    echo >&2 "  - Windows: https://docs.docker.com/desktop/setup/install/windows-install/"
+    echo >&2 ""
+    echo >&2 "After installing Docker, make sure it's running and try again."
+    echo >&2 ""
+    exit 1
+}
+
 # ── Step 1: dependency check ────────────────────────────────────────
 log "checking dependencies"
-need docker
+check_docker
 need curl
 need openssl
 docker compose version >/dev/null 2>&1 \

package.json

@@ -54,7 +54,6 @@
     "dev": "npm start",
     "build": "npm run setupExtensions && npm run build:ts && cd src/gui && node ./build.js && cd ../puter-js && npm run build",
     "build:workerLib": "cd src/puter-js && npm run build && cd ../worker && npm run build",
-    "check-translations": "node tools/check-translations.js",
     "prepare": "husky",
     "build:ts": "tsc -p tsconfig.build.json && node ./tools/write-dist-package-json.mjs",
     "check:puterjs:types": "node tools/checkPuterjsTypes.mjs",
@@ -79,7 +78,8 @@
     "javascript-time-ago": "^2.5.11",
     "libphonenumber-js": "1.13.15",
     "miniflare": "^4.20260617.1",
-    "open": "^10.1.0"
+    "open": "^10.1.0",
+    "sharp": "^0.35.5"
   },
   "engines": {
     "node": ">=24.0.0"

src/backend/clients/database/DatabaseClient.test.ts

@@ -275,6 +275,28 @@ describe('AbstractDatabaseClient — SQL generation', () => {
         expect(postgres().timestampFromUnixSql()).toBe('to_timestamp(?)');
     });
 
+    it('reads a timestamp column back as unix seconds in each engine', () => {
+        expect(sqlite().unixSecondsSql('`t`')).toBe(
+            "CAST(strftime('%s', `t`) AS INTEGER)",
+        );
+        expect(mysql().unixSecondsSql('`t`')).toBe('UNIX_TIMESTAMP(`t`)');
+        expect(postgres().unixSecondsSql('`t`')).toBe(
+            'EXTRACT(EPOCH FROM `t`)::bigint',
+        );
+    });
+
+    it('reads the database clock as epoch milliseconds in each dialect', () => {
+        expect(sqlite().nowMsExpr()).toBe(
+            "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+        );
+        expect(mysql().nowMsExpr()).toBe(
+            "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        );
+        expect(postgres().nowMsExpr()).toBe(
+            'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+        );
+    });
+
     it('describes a stored timestamp the way each driver reads it back', () => {
         const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
         expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');

src/backend/clients/database/DatabaseClient.ts

@@ -388,6 +388,34 @@ export class AbstractDatabaseClient extends PuterClient {
         });
     }
 
+    /**
+     * A timestamp column as unix seconds, computed by the database: the mysql
+     * driver reads a stored UTC timestamp as local time, so parsing it in JS
+     * can be off by the server's UTC offset.
+     */
+    unixSecondsSql(column: string): string {
+        return this.case({
+            postgres: `EXTRACT(EPOCH FROM ${column})::bigint`,
+            mysql: `UNIX_TIMESTAMP(${column})`,
+            otherwise: `CAST(strftime('%s', ${column}) AS INTEGER)`,
+        });
+    }
+
+    /**
+     * The database's own clock as epoch milliseconds, so nodes whose clocks
+     * disagree still agree on it. The mysql form ignores the session time zone,
+     * so a DST change can't shift it.
+     */
+    nowMsExpr(): string {
+        return this.case({
+            sqlite: "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+            postgres:
+                'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+            otherwise:
+                "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        });
+    }
+
     /**
      * What a read of a timestamp column holding `seconds` returns: sqlite gives
      * back the UTC text it stored, mysql2 and pg a `Date` read in node's time

src/backend/clients/database/SqliteDatabaseClient.test.ts

@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
 import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
 
 /** Highest schema version the migration table can reach. */
-const CURRENT_SCHEMA_VERSION = 93;
+const CURRENT_SCHEMA_VERSION = 96;
 
 /**
  * These suites migrate real files on disk. Idle they finish in well under a
@@ -673,6 +673,38 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
         });
     });
 
+    it('adds kv_share_handles.notify from 0099, off for every existing share', async () => {
+        const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-notify-'));
+        const path = join(dir, 'puter.sqlite');
+        try {
+            const before = new SqliteDatabaseClient(
+                sqliteConfig({ inMemory: false, path, targetVersion: 94 }),
+            );
+            await before.onServerStart();
+            await before.write(
+                'INSERT INTO `kv_share_handles` (`handle`, `owner_user_id`, ' +
+                    '`grantee_user_id`, `app_uid`, `key_prefix`, `permission`, ' +
+                    '`created_at`) VALUES (?, 1, 1, ?, ?, ?, 0)',
+                ['kvh-before-notify', 'os-global', 'a:', 'kv-share:x'],
+            );
+            before.onServerShutdown();
+
+            const after = new SqliteDatabaseClient(
+                sqliteConfig({ inMemory: false, path }),
+            );
+            await after.onServerStart();
+            await expect(
+                after.read(
+                    'SELECT `notify` FROM `kv_share_handles` WHERE `handle` = ?',
+                    ['kvh-before-notify'],
+                ),
+            ).resolves.toEqual([{ notify: 0 }]);
+            after.onServerShutdown();
+        } finally {
+            rmSync(dir, { recursive: true, force: true });
+        }
+    });
+
     it('indexes notification.created_at for the retention sweep', async () => {
         const rows = await client.read('PRAGMA index_list(`notification`)');
         expect(rows.map((r) => String(r.name))).toContain(
@@ -680,6 +712,29 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
         );
     });
 
+    it('indexes notification.replaced_at per mailbox slice for catch-up', async () => {
+        const columns = await client.read(
+            'PRAGMA index_info(`idx_notification_replaced`)',
+        );
+        expect(columns.map((c) => String(c.name))).toEqual([
+            'user_id',
+            'audience',
+            'app_uid',
+            'replaced_at',
+        ]);
+    });
+
+    it('keeps notification.send_key unique', async () => {
+        const rows = await client.read('PRAGMA index_list(`notification`)');
+        expect(
+            rows.find((r) => r.name === 'idx_notification_send_key'),
+        ).toMatchObject({ unique: 1 });
+        const columns = await client.read(
+            'PRAGMA index_info(`idx_notification_send_key`)',
+        );
+        expect(columns.map((c) => String(c.name))).toEqual(['send_key']);
+    });
+
     it('leaves an already-migrated database untouched on a second boot', async () => {
         const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-'));
         const path = join(dir, 'nested', 'puter.sqlite');

src/backend/clients/database/SqliteDatabaseClient.ts

@@ -132,6 +132,9 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
     [90, ['0095_kv-share-handle-levels.sql']],
     [91, ['0096_kv-share-handle-quota-indexes.sql']],
     [92, ['0097_user-to-user-holder-permission-index.sql']],
+    [93, ['0098_notification-groups.sql']],
+    [94, ['0099_kv-share-handle-notify.sql']],
+    [95, ['0100_notification-send-key.sql']],
 ];
 
 export class SqliteDatabaseClient extends AbstractDatabaseClient {

src/backend/clients/database/migrations/mysql/mysql_mig_52.sql

@@ -0,0 +1,53 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`group_key` and `replaced_at`. See
+-- sqlite/0098_notification-groups.sql for the rationale. The key is a hex
+-- digest, so ascii and compared byte for byte. No per-file applied-state
+-- tracking, so every step is guarded and a replay does nothing.
+
+CALL _puter_add_col('notification', 'group_key', '`group_key` char(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
+CALL _puter_add_col('notification', 'replaced_at', '`replaced_at` bigint DEFAULT NULL');
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_group_indexes;
+DELIMITER //
+CREATE PROCEDURE _puter_add_notification_group_indexes()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_group'
+  ) THEN
+    ALTER TABLE `notification` ADD INDEX `idx_notification_group`
+      (`user_id`, `app_uid`, `group_key`);
+  END IF;
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_replaced'
+  ) THEN
+    ALTER TABLE `notification` ADD INDEX `idx_notification_replaced`
+      (`user_id`, `audience`, `app_uid`, `replaced_at`);
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_notification_group_indexes();
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_group_indexes;

src/backend/clients/database/migrations/mysql/mysql_mig_53.sql

@@ -0,0 +1,23 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Notifications through a share handle. See
+-- sqlite/0099_kv-share-handle-notify.sql for the column. No per-file
+-- applied-state tracking, so it goes through _puter_add_col and a replay does
+-- nothing.
+
+CALL _puter_add_col('kv_share_handles', 'notify', '`notify` tinyint(1) NOT NULL DEFAULT 0');

src/backend/clients/database/migrations/mysql/mysql_mig_54.sql

@@ -0,0 +1,43 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`. See sqlite/0100_notification-send-key.sql for the
+-- rationale. The key is a uuid, so ascii and compared byte for byte. No
+-- per-file applied-state tracking, so every step is guarded and a replay does
+-- nothing.
+
+CALL _puter_add_col('notification', 'send_key', '`send_key` char(36) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
+DELIMITER //
+CREATE PROCEDURE _puter_add_notification_send_key_index()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_send_key'
+  ) THEN
+    ALTER TABLE `notification` ADD UNIQUE KEY `idx_notification_send_key`
+      (`send_key`), ALGORITHM=INPLACE, LOCK=NONE;
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_notification_send_key_index();
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;

src/backend/clients/database/migrations/postgres/postgres_mig_41.sql

@@ -0,0 +1,29 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- notification.group_key and replaced_at. See
+-- sqlite/0098_notification-groups.sql for the rationale. Idempotent via IF NOT
+-- EXISTS.
+
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS group_key varchar(64) DEFAULT NULL;
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS replaced_at bigint DEFAULT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_notification_group
+    ON notification (user_id, app_uid, group_key);
+
+CREATE INDEX IF NOT EXISTS idx_notification_replaced
+    ON notification (user_id, audience, app_uid, replaced_at);

src/backend/clients/database/migrations/postgres/postgres_mig_42.sql

@@ -0,0 +1,22 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Notifications through a share handle. See
+-- sqlite/0099_kv-share-handle-notify.sql for the column. Idempotent via
+-- IF NOT EXISTS; there is no per-file applied-state tracking.
+
+ALTER TABLE kv_share_handles ADD COLUMN IF NOT EXISTS notify smallint NOT NULL DEFAULT 0;

src/backend/clients/database/migrations/postgres/postgres_mig_43.sql

@@ -0,0 +1,24 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- notification.send_key. See sqlite/0100_notification-send-key.sql for the
+-- rationale. Idempotent via IF NOT EXISTS.
+
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS send_key char(36) DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_send_key
+    ON notification (send_key);

src/backend/clients/database/migrations/sqlite/0098_notification-groups.sql

@@ -0,0 +1,38 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- What app message groups need from `notification`:
+--
+--   - `group_key`   : a sha256 hex digest of an app message's `group`, so the
+--                     notification a newer one replaces is found by index.
+--                     Null on every other row, and on rows written before
+--                     this, which therefore never match.
+--   - `replaced_at` : epoch milliseconds of the last replacement, null for a
+--                     row never replaced. Live and fetched copies carry it as
+--                     their version, so a client can tell a replacement from a
+--                     copy it already has. Indexed per mailbox slice, so a
+--                     catch-up reads the rows replaced since its cursor
+--                     without scanning the ones that never were.
+
+ALTER TABLE `notification` ADD COLUMN `group_key` TEXT DEFAULT NULL;
+ALTER TABLE `notification` ADD COLUMN `replaced_at` INTEGER DEFAULT NULL;
+
+CREATE INDEX IF NOT EXISTS `idx_notification_group`
+    ON `notification` (`user_id`, `app_uid`, `group_key`);
+
+CREATE INDEX IF NOT EXISTS `idx_notification_replaced`
+    ON `notification` (`user_id`, `audience`, `app_uid`, `replaced_at`);

src/backend/clients/database/migrations/sqlite/0099_kv-share-handle-notify.sql

@@ -0,0 +1,23 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Whether the two parties of a share may send each other notifications through
+-- its handle. Off by default, so every existing share reads as not allowing
+-- them.
+
+ALTER TABLE `kv_share_handles`
+    ADD COLUMN `notify` INTEGER NOT NULL DEFAULT 0;

src/backend/clients/database/migrations/sqlite/0100_notification-send-key.sql

@@ -0,0 +1,27 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`: the key of the delayed send (a `deferred`
+-- schedule) that wrote this row or last replaced it, derived from the
+-- schedule and its slot. Unique, so one slot can never write two rows, and a
+-- send that crashed after writing finds its row again by it. Null on every row
+-- a delayed send never touched.
+
+ALTER TABLE `notification` ADD COLUMN `send_key` TEXT DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS `idx_notification_send_key`
+    ON `notification` (`send_key`);

src/backend/clients/event/types.ts

@@ -75,6 +75,15 @@ export type EventMap = {
     serverStart: Record<string, never>;
     serverPrepareShutdown: Record<string, never>;
     serverShutdown: Record<string, never>;
+    /**
+     * Emitted by a host extension that knows this server will restart soon.
+     * Long-lived connections ask their clients to reconnect; nothing is refused
+     * or closed, so a false alarm is harmless.
+     */
+    serverRestartAnnounced: {
+        /** How long until the server stops, when known. */
+        closeInMs?: number;
+    };
 
     // ---- AI ----
     'ai.prompt.validate': {
@@ -314,6 +323,8 @@ export type EventMap = {
         // the abuse extension cap sends per device (across accounts) so a
         // device farm can't dodge the per-account cap with fresh signups.
         device_fingerprint: string | null;
+        // Browser-signals dispatch id the client sent, or null.
+        dispatch_id: string | null;
         allowed: boolean;
         reason: string | null;
         // With `allowed` false, offer the card fallback in place of SMS. Only
@@ -549,6 +560,8 @@ export type EventMap = {
         appUid: string | null;
         value: Record<string, unknown>;
         createdAt: number;
+        /** An earlier notification replaced in place; `createdAt` is when. */
+        replaced?: true;
         /** Events handler runs behind whatever created it. */
         handlerDepth?: number;
     };
@@ -568,6 +581,8 @@ export type EventMap = {
         updated?: true;
         /** Names this one send, for a receipt asked about it. */
         sendId: string;
+        /** The same `sendId` may be emitted again, as a delayed send is. */
+        replayable?: true;
     };
     /**
      * One notification, addressed at whatever sockets each region holds for the
@@ -577,7 +592,7 @@ export type EventMap = {
      */
     'outer.notif.delivery': {
         userId: number;
-        wire: 'notif.message' | 'notif.unreads' | 'notif.ack';
+        wire: 'notif.message' | 'notif.unreads' | 'notif.ack' | 'notif.removed';
         response: unknown;
     };
     /**
@@ -743,6 +758,7 @@ export type EventMap = {
     'outer.gui.item.renamed': GuiEvent;
     'outer.gui.notif.ack': GuiEvent<{ uid: string }>;
     'outer.gui.notif.message': GuiEvent<{ uid: string; notification: unknown }>;
+    'outer.gui.notif.removed': GuiEvent<{ uid: string }>;
     'outer.gui.notif.unreads': GuiEvent<{
         unreads: { uid: string; notification: unknown }[];
     }>;
@@ -938,6 +954,9 @@ export type AiCostFactorEvent = {
  */
 export type LifecyclePhase = 'before' | 'after' | 'error' | 'reject';
 
+/** Status a vetoing `before` listener may answer with. */
+export type LifecycleRejectStatus = 403 | 429;
+
 /**
  * Payload for `driver.<iface>.<method>.<phase>` events.
  *
@@ -967,6 +986,8 @@ export type DriverMethodLifecycleEvent = {
     allow?: boolean;
     /** Optional human-readable reason surfaced to the caller when vetoed. */
     rejectReason?: string;
+    /** Status for a veto: 403 (default) or 429 for a throttle. */
+    rejectStatus?: LifecycleRejectStatus;
 };
 
 /**
@@ -1001,6 +1022,8 @@ export type RouteLifecycleEvent = {
     allow?: boolean;
     /** Optional human-readable reason surfaced to the caller when vetoed. */
     rejectReason?: string;
+    /** Status for a veto: 403 (default) or 429 for a throttle. */
+    rejectStatus?: LifecycleRejectStatus;
 };
 
 export type EventKey = keyof EventMap & string;
@@ -1009,10 +1032,11 @@ export type EventKey = keyof EventMap & string;
 // Generates a wildcard for every non-final dot-separated prefix of K.
 export type WildcardPrefixes<K extends string> =
     K extends `${infer Head}.${infer Tail}`
-        ? | `${Head}.*`
-          | (Tail extends `${string}.${string}`
-                ? `${Head}.${WildcardPrefixes<Tail>}`
-                : never)
+        ?
+              | `${Head}.*`
+              | (Tail extends `${string}.${string}`
+                    ? `${Head}.${WildcardPrefixes<Tail>}`
+                    : never)
         : never;
 
 export type ListenKey = EventKey | WildcardPrefixes<EventKey>;
@@ -1023,11 +1047,21 @@ export type MatchingEvents<P extends ListenKey> = P extends `${infer Prefix}.*`
 
 /** Acked, not acked, no socket anywhere, or sockets only in a peer region. */
 export type NotifDeliveryOutcome =
-    'received' | 'unreceived' | 'no-socket' | 'remote-only';
+    | 'received'
+    | 'unreceived'
+    | 'no-socket'
+    | 'remote-only';
 
 /** One socket's receipt for a live notification. */
 export interface NotifDeliveryResponder {
     visibilityState: 'visible' | 'hidden' | 'unknown';
+    /** The page's window has focus. Absent from older clients. */
+    focused?: boolean;
+    /**
+     * The page shows its own browser notification while hidden or unfocused.
+     * Absent from older clients.
+     */
+    showsOwn?: boolean;
 }
 
 export type EventMetadata = {

src/backend/clients/events/EventsWorkerInvokerClient.ts

@@ -43,6 +43,20 @@ import { PuterClient } from '../types.js';
  *
  * On a miss, a handler set here may deploy the script itself and retry once
  * with `x-puter-events-deployed: 1`, skipping the dispatcher's own callback.
+ *
+ * Calls that come due within a few milliseconds of each other go out together:
+ *
+ *     POST <events dispatcher>/invoke-batch
+ *     x-puter-internal-auth:  <internal secret>
+ *     { calls: [{ script, appUid, key, deployed?, body, waitMs }] }
+ *
+ * The dispatcher runs a few at a time, in order, and streams NDJSON: `{ i,
+ * started: true }` when a call goes to its worker, then `{ i, status, handled?,
+ * error?, retryAfter?, dispatch? }` when it finishes — the same facts the
+ * single route carries in its status and headers. A call's timeout runs from
+ * its `started` line, so time spent queued behind other calls is never the
+ * handler's. A call still queued `waitMs` after the batch arrived is never
+ * started and answers `dispatch: "queue-timeout"`.
  */
 
 /** The one route an events worker answers. */
@@ -51,6 +65,27 @@ export const EVENTS_INVOKE_PATH = '/__events/invoke';
 /** The route the events dispatcher takes an invocation on. */
 export const EVENTS_DISPATCH_PATH = '/invoke';
 
+/** The route it takes several on, answering each as it finishes. */
+export const EVENTS_DISPATCH_BATCH_PATH = '/invoke-batch';
+
+/** Calls one batch carries at most. */
+export const INVOKE_BATCH_SIZE = 60;
+
+/** How long the first call of a batch waits for others to join it. */
+export const INVOKE_BATCH_WINDOW_MS = 5;
+
+/**
+ * How long a dispatcher without the batch route is sent single calls before the
+ * batch route is tried again.
+ */
+const BATCH_UNSUPPORTED_RETRY_MS = 5 * 60_000;
+
+/**
+ * How long past its wait a batched call's `started` line may take to arrive
+ * before the call is given up as never started.
+ */
+const BATCH_START_GRACE_MS = 2_000;
+
 /**
  * Header the dispatcher marks its own failures with. Anything carrying it never
  * reached a handler, so its status says nothing about the delivery.
@@ -68,6 +103,23 @@ export const EVENTS_HANDLED_HEADER = 'x-puter-events-handled';
 /** Machine-readable reason on the runtime's own failure answers. */
 export const EVENTS_ERROR_HEADER = 'x-puter-events-error';
 
+/**
+ * The runtime's answer to a key it does not hold: the script was deployed under
+ * a secret that has since rotated, and redeploying it binds the current key.
+ */
+const BAD_KEY_ERROR = 'bad-key';
+
+/**
+ * Runtime failure answers that are Puter's fault rather than the handler's: a
+ * key the script no longer matches, no token to run as, or a handler that
+ * failed because a Puter API call it made did.
+ */
+const PLATFORM_ERROR_CODES = new Set([
+    BAD_KEY_ERROR,
+    'no-token',
+    'platform-error',
+]);
+
 /**
  * Set on the retry after this backend deploys a missing script itself, so the
  * dispatcher skips its own deploy-on-miss callback and negative cache and goes
@@ -95,10 +147,16 @@ const SELF_DEPLOYABLE_MISS_REASONS = new Set([
  */
 const ALREADY_DEPLOYED_MISS_REASONS = new Set(['deploy-timeout']);
 
+/** Whose call asked for the deploy, for the share of deploys it may take. */
+export interface WorkerMissOptions {
+    lane?: InvokeLane;
+}
+
 /** What a self-deploy attempt did, and to which app/script it applies. */
 export type WorkerMissHandler = (
     appUid: string,
     script: string,
+    options: WorkerMissOptions,
 ) => Promise<'deployed' | string>;
 
 /** How long a handler has to answer before the attempt is abandoned. */
@@ -106,12 +164,31 @@ export const EVENTS_INVOKE_TIMEOUT_MS = 30_000;
 
 /**
  * Connections held open to the dispatcher, so a busy fleet is not
- * re-handshaking. The 30s invoke abort starts when a request is queued, not
- * when it gets a connection, so a saturated pool shows up as timeouts.
+ * re-handshaking. Also how many calls one transport makes at once: a call past
+ * that waits for a free one before its invoke timeout starts.
  */
-const INVOKE_CONNECTIONS = 256;
+export const INVOKE_CONNECTIONS = 256;
+
+/**
+ * Of those, how many schedule runs may hold at once, so a burst of schedules
+ * can never take the connections event deliveries need.
+ */
+export const SCHEDULE_INVOKE_CONNECTIONS = 64;
+
+/** Whose call this is, for the connection share it may take. */
+export type InvokeLane = 'delivery' | 'schedule';
+
+/** What a deploy answers when another deploy of the region holds the way. */
+export const DEPLOY_BUSY = 'busy';
 const INVOKE_KEEP_ALIVE_MS = 30_000;
 
+/**
+ * A call waits at most this share of its invoke timeout for a connection, so
+ * the wait and the run together stay inside the worker lease (twice the
+ * timeout). Past it the call fails as Puter's, not the handler's.
+ */
+const CONNECTION_WAIT_SHARE = 0.5;
+
 /** One invocation, as the events service hands it over. */
 export interface WorkerInvokeRequest {
     /** Script the app's current handler set deploys as. */
@@ -137,36 +214,69 @@ export interface EventsInvokeCall {
     timeoutMs: number;
     /** Set on the retry after this backend deployed the script itself. */
     deployed?: boolean;
+    lane?: InvokeLane;
+    /**
+     * Builds `body` again at send time, for a transport that waits for a
+     * connection first, so a send time stamped into it is the real one.
+     */
+    render?: () => string;
+}
+
+/** What one attempt at delivering an invocation came back with. */
+export interface EventsInvokeAnswer {
+    /** `null` when no answer came back. */
+    status: number | null;
+    /** Set only when the answer provably came from the script. */
+    handled?: boolean;
+    error?: string;
+    /** Set only by the dispatcher transport, naming its own failure. */
+    dispatchReason?: string;
+    /** No answer within the invoke timeout. */
+    timedOut?: boolean;
+    /** From a `retry-after` header in seconds. */
+    retryAfterMs?: number;
+    /** A deploy this call needed was put off: another held the way. */
+    deployBusy?: boolean;
 }
 
 /**
- * Delivers an invocation. `status` null means nothing ran (retriable);
- * `handled` is set only when the answer provably came from the script, so an
- * unmarked 4xx is not a handler's refusal.
+ * Delivers an invocation. `handled` is set only when the answer provably came
+ * from the script, so an unmarked 4xx is not a handler's refusal.
  */
 export interface EventsInvokeTransport {
-    send(call: EventsInvokeCall): Promise<{
-        status: number | null;
-        handled?: boolean;
-        error?: string;
-        /** Set only by the dispatcher transport, naming its own failure. */
-        dispatchReason?: string;
-    }>;
+    send(call: EventsInvokeCall): Promise<EventsInvokeAnswer>;
 }
 
 /**
  * `settled`: handler accepted (2xx). `terminal`: handler refused (marked 4xx),
- * not retried. `retriable`: no handler answer (5xx, 429, timeout, transport
- * failure, unmarked 4xx).
+ * not retried. `retriable`: the handler failed or ran past the timeout.
+ * `unavailable`: Puter could not start or finish the run (transport, dispatch
+ * and deploy failures, unmarked answers, a Puter API failure inside the
+ * handler); retried like `retriable` but never the handler's fault.
  */
-export type WorkerInvokeOutcome = 'settled' | 'terminal' | 'retriable';
+export type WorkerInvokeOutcome =
+    | 'settled'
+    | 'terminal'
+    | 'retriable'
+    | 'unavailable';
 
 export interface WorkerInvokeResult {
     outcome: WorkerInvokeOutcome;
+    /** A deploy on miss was put off because another one held the way. */
+    deployBusy?: boolean;
     /** The status the handler answered with, or `null` when it never did. */
     status: number | null;
     /** Why it could not answer, when nothing did. */
     error?: string;
+    /** How long the answer asked to be left alone. */
+    retryAfterMs?: number;
+}
+
+/** How one invocation is sent. */
+export interface WorkerInvokeOptions {
+    lane?: InvokeLane;
+    /** Applied to the event as each attempt is sent. */
+    stampEvent?: (event: unknown) => unknown;
 }
 
 const NO_TRANSPORT =
@@ -187,8 +297,8 @@ export class EventsWorkerInvokerClient extends PuterClient {
     /**
      * Deploys a script the dispatcher reports missing, in place of the
      * rehydrate callback — for the case where that callback lands on a backend
-     * other than this one. Set once; `invoke()` calls it at most once per
-     * delivery and never loops.
+     * other than this one — or one still bound to a rotated key. Set once;
+     * `invoke()` calls it at most once per delivery and never loops.
      */
     setMissHandler(handler: WorkerMissHandler): void {
         this.#missHandler = handler;
@@ -199,67 +309,92 @@ export class EventsWorkerInvokerClient extends PuterClient {
         this.#dispatcher = null;
     }
 
-    async invoke(request: WorkerInvokeRequest): Promise<WorkerInvokeResult> {
+    async invoke(
+        request: WorkerInvokeRequest,
+        options: WorkerInvokeOptions = {},
+    ): Promise<WorkerInvokeResult> {
         const transport = this.#pickTransport();
         if (!transport)
-            return { outcome: 'retriable', status: null, error: NO_TRANSPORT };
+            return {
+                outcome: 'unavailable',
+                status: null,
+                error: NO_TRANSPORT,
+            };
 
-        const call: EventsInvokeCall = {
-            script: request.script,
-            appUid: request.appUid,
-            key: request.key,
-            body: JSON.stringify({
+        const { stampEvent } = options;
+        const render = (): string =>
+            JSON.stringify({
                 handler: request.handler,
-                event: request.event,
+                event: stampEvent ? stampEvent(request.event) : request.event,
                 ctx: request.ctx,
                 token: request.token,
-            }),
-            timeoutMs: this.#timeoutMs(),
-        };
+            });
+        const send = (deployed: boolean): Promise<EventsInvokeAnswer> =>
+            transport.send({
+                script: request.script,
+                appUid: request.appUid,
+                key: request.key,
+                body: render(),
+                timeoutMs: this.#timeoutMs(),
+                ...(deployed ? { deployed: true } : {}),
+                ...(options.lane ? { lane: options.lane } : {}),
+                ...(stampEvent ? { render } : {}),
+            });
 
-        let result = await transport.send(call);
-        if (result.status === null && result.dispatchReason) {
-            if (
-                this.#missHandler &&
-                SELF_DEPLOYABLE_MISS_REASONS.has(result.dispatchReason)
-            ) {
-                const outcome = await this.#missHandler(
-                    request.appUid,
-                    request.script,
-                );
-                if (outcome !== 'deployed')
-                    return {
-                        outcome: 'retriable',
-                        status: null,
-                        error: `deploy: ${outcome}`,
-                    };
-                result = await transport.send({ ...call, deployed: true });
-            } else if (
-                ALREADY_DEPLOYED_MISS_REASONS.has(result.dispatchReason)
-            ) {
-                // No deploy to trigger — just give propagation the one retry
-                // it usually needs, without spending an upload on it.
-                result = await transport.send({ ...call, deployed: true });
-            }
+        let result = await send(false);
+        if (this.#missHandler && selfDeployable(result)) {
+            const outcome = await this.#missHandler(
+                request.appUid,
+                request.script,
+                { lane: options.lane },
+            );
+            if (outcome !== 'deployed')
+                return {
+                    outcome: 'unavailable',
+                    status: null,
+                    error: `deploy: ${outcome}`,
+                    ...(outcome === DEPLOY_BUSY ? { deployBusy: true } : {}),
+                };
+            result = await send(true);
+        } else if (
+            result.status === null &&
+            result.dispatchReason &&
+            ALREADY_DEPLOYED_MISS_REASONS.has(result.dispatchReason)
+        ) {
+            // No deploy to trigger — just give propagation the one retry
+            // it usually needs, without spending an upload on it.
+            result = await send(true);
         }
 
-        const { status, handled, error } = result;
+        const { status, handled, error, timedOut, retryAfterMs } = result;
+        if (result.deployBusy)
+            return {
+                outcome: 'unavailable',
+                status: null,
+                error,
+                deployBusy: true,
+            };
+        // Past the timeout the handler was most likely still running, which
+        // is its own doing; any other missing answer never reached it.
         if (status === null)
-            return { outcome: 'retriable', status: null, error };
+            return {
+                outcome: timedOut ? 'retriable' : 'unavailable',
+                status: null,
+                error,
+            };
 
-        const outcome = outcomeForStatus(status, handled === true);
-        // An unmarked 4xx (429 aside — that one is always retriable) is not
-        // the handler's own answer; say so even when the transport did not,
-        // so the reason a delivery kept retrying shows up in the log rather
-        // than reading as an unexplained retry.
-        const unmarkedFourxx =
-            status !== 429 && status >= 400 && status < 500 && !handled;
+        const outcome = outcomeForStatus(status, handled === true, error);
+        // An unmarked failure is not the handler's own answer; say so even
+        // when the transport did not, so the reason a delivery kept retrying
+        // shows up in the log rather than reading as an unexplained retry.
+        const unmarked = status >= 300 && !handled;
         const reportedError =
-            error ?? (unmarkedFourxx ? 'unmarked 4xx' : undefined);
+            error ?? (unmarked ? `unmarked ${status}` : undefined);
         return {
             outcome,
             status,
             ...(reportedError ? { error: reportedError } : {}),
+            ...(retryAfterMs ? { retryAfterMs } : {}),
         };
     }
 
@@ -270,9 +405,19 @@ export class EventsWorkerInvokerClient extends PuterClient {
         // a misconfiguration worth reporting once per attempt rather than a
         // round trip.
         if (!dispatcherUrl || !internalSecret) return null;
+        const { invokeBatchSize, invokeBatchWindowMs } =
+            this.config.events ?? {};
         this.#dispatcher ??= new DispatcherInvokeTransport(
             dispatcherUrl,
             internalSecret,
+            {
+                ...(invokeBatchSize !== undefined
+                    ? { batchSize: invokeBatchSize }
+                    : {}),
+                ...(invokeBatchWindowMs !== undefined
+                    ? { batchWindowMs: invokeBatchWindowMs }
+                    : {}),
+            },
         );
         return this.#dispatcher;
     }
@@ -285,52 +430,132 @@ export class EventsWorkerInvokerClient extends PuterClient {
     }
 }
 
+/** A call waiting in, or sent with, the current batch. */
+interface BatchedCall {
+    call: EventsInvokeCall;
+    resolve: (answer: EventsInvokeAnswer) => void;
+    /** When its wait for a connection began, for what is left of it. */
+    waitingSince: number;
+}
+
+export interface DispatcherInvokeTransportOptions {
+    fetchImpl?: typeof undiciFetch;
+    /** Calls per batch; 1 sends every call on its own. */
+    batchSize?: number;
+    batchWindowMs?: number;
+}
+
 /** Reaches an events worker the only way production can: through the dispatcher. */
 export class DispatcherInvokeTransport implements EventsInvokeTransport {
     readonly #url: string;
     readonly #secret: string;
     readonly #fetchImpl: typeof undiciFetch;
+    readonly #batchSize: number;
+    readonly #batchWindowMs: number;
     #agent: Agent | null = null;
+    /** Calls holding a connection slot. */
+    #active = 0;
+    /** Of those, schedule runs. */
+    #scheduleActive = 0;
+    /** Calls waiting for one, longest-waiting first, with their lane. */
+    readonly #waiting = new Map<() => void, InvokeLane>();
+    /** Calls gathered for the next batch. */
+    #batch: BatchedCall[] = [];
+    #batchTimer: ReturnType<typeof setTimeout> | null = null;
+    /** Until when the dispatcher is taken not to have the batch route. */
+    #batchUnsupportedUntil = 0;
 
     constructor(
         dispatcherUrl: string,
         secret: string,
-        options: { fetchImpl?: typeof undiciFetch } = {},
+        options: DispatcherInvokeTransportOptions = {},
     ) {
-        this.#url = dispatcherUrl;
+        // Joined as a string: `new URL(path, base)` resolves an absolute path
+        // against the base's origin, dropping any path prefix it carries.
+        this.#url = dispatcherUrl.replace(/\/$/, '');
         this.#secret = secret;
         this.#fetchImpl = options.fetchImpl ?? undiciFetch;
+        this.#batchSize = Math.max(
+            1,
+            Math.floor(options.batchSize ?? INVOKE_BATCH_SIZE),
+        );
+        this.#batchWindowMs = Math.max(
+            0,
+            options.batchWindowMs ?? INVOKE_BATCH_WINDOW_MS,
+        );
     }
 
     close(): void {
+        this.#flush();
         void this.#agent?.close().catch(() => {});
         this.#agent = null;
     }
 
-    async send(call: EventsInvokeCall): Promise<{
-        status: number | null;
-        handled?: boolean;
-        error?: string;
-        dispatchReason?: string;
-    }> {
+    async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        // No `timedOut`: the handler never got this call, so the wait is not
+        // its failure.
+        const lane = call.lane ?? 'delivery';
+        const waitingSince = Date.now();
+        if (
+            !(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE, lane))
+        )
+            return {
+                status: null,
+                error: 'no free connection to the events dispatcher',
+            };
         try {
-            // `new URL(path, base)` resolves an absolute path against the
-            // base's origin, dropping any path prefix `base` already carries.
-            const url = `${this.#url.replace(/\/$/, '')}${EVENTS_DISPATCH_PATH}`;
-            const response = await this.#fetchImpl(url, {
-                method: 'POST',
-                headers: {
-                    'content-type': 'application/json',
-                    'x-puter-internal-auth': this.#secret,
-                    'x-puter-events-script': call.script,
-                    'x-puter-events-app': call.appUid,
-                    'x-puter-events-key': call.key,
-                    ...(call.deployed ? { [EVENTS_DEPLOYED_HEADER]: '1' } : {}),
-                },
-                body: call.body,
-                signal: AbortSignal.timeout(call.timeoutMs),
-                dispatcher: this.#dispatcherFor(),
+            if (this.#batchSize === 1) return await this.#sendOne(call);
+            return await new Promise<EventsInvokeAnswer>((resolve) => {
+                this.#batch.push({ call, resolve, waitingSince });
+                if (this.#batch.length >= this.#batchSize) this.#flush();
+                else
+                    this.#batchTimer ??= setTimeout(
+                        () => this.#flush(),
+                        this.#batchWindowMs,
+                    );
             });
+        } finally {
+            this.#release(lane);
+        }
+    }
+
+    /** Send what has gathered: alone when it is one call, else as a batch. */
+    #flush(): void {
+        if (this.#batchTimer) clearTimeout(this.#batchTimer);
+        this.#batchTimer = null;
+        const batch = this.#batch;
+        this.#batch = [];
+        if (batch.length === 0) return;
+        if (batch.length === 1 || Date.now() < this.#batchUnsupportedUntil) {
+            for (const { call, resolve } of batch)
+                void this.#sendOne(call).then(resolve);
+            return;
+        }
+        void this.#sendBatch(batch);
+    }
+
+    /** One call on the single route. */
+    async #sendOne(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                        'x-puter-events-script': call.script,
+                        'x-puter-events-app': call.appUid,
+                        'x-puter-events-key': call.key,
+                        ...(call.deployed
+                            ? { [EVENTS_DEPLOYED_HEADER]: '1' }
+                            : {}),
+                    },
+                    body: call.render ? call.render() : call.body,
+                    signal: AbortSignal.timeout(call.timeoutMs),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
             // The body is not read: a handler answers with a status, and
             // whatever else it writes is its own business.
             void response.body?.cancel().catch(() => {});
@@ -341,24 +566,210 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
                 EVENTS_DISPATCH_ERROR_HEADER,
             );
             if (dispatchError !== null)
-                return {
-                    status: null,
-                    error: `dispatcher: ${dispatchError} (${response.status})`,
-                    dispatchReason: dispatchError,
-                };
+                return dispatchAnswer(dispatchError, response.status);
 
-            const handled = response.headers.get(EVENTS_HANDLED_HEADER) === '1';
-            const errorHeader = response.headers.get(EVENTS_ERROR_HEADER);
-            return {
-                status: response.status,
-                handled,
-                ...(errorHeader ? { error: errorHeader } : {}),
-            };
+            return answerFrom(response);
         } catch (err) {
-            return {
+            return failedAnswer(err);
+        }
+    }
+
+    /**
+     * Several calls in one request. A call may wait in the dispatcher's queue
+     * for whatever is left of its connection wait, and its timeout runs from
+     * its `started` line; the request itself is dropped once every call has an
+     * answer.
+     */
+    async #sendBatch(batch: BatchedCall[]): Promise<void> {
+        const hangUp = new AbortController();
+        let open = batch.length;
+        const answered = batch.map(() => false);
+        const timers: ReturnType<typeof setTimeout>[] = [];
+        const settle = (i: number, answer: EventsInvokeAnswer): void => {
+            if (answered[i]) return;
+            answered[i] = true;
+            clearTimeout(timers[i]);
+            batch[i].resolve(answer);
+            if (--open === 0) hangUp.abort();
+        };
+        const settleRest = (answer: EventsInvokeAnswer): void => {
+            for (let i = 0; i < batch.length; i++) settle(i, answer);
+        };
+
+        const now = Date.now();
+        const waits = batch.map(({ call, waitingSince }) =>
+            Math.max(
+                0,
+                Math.floor(
+                    call.timeoutMs * CONNECTION_WAIT_SHARE -
+                        (now - waitingSince),
+                ),
+            ),
+        );
+        const body = JSON.stringify({
+            calls: batch.map(({ call }, i) => ({
+                script: call.script,
+                appUid: call.appUid,
+                key: call.key,
+                ...(call.deployed ? { deployed: true } : {}),
+                body: call.render ? call.render() : call.body,
+                waitMs: waits[i],
+            })),
+        });
+        const arm = (
+            i: number,
+            ms: number,
+            answer: () => EventsInvokeAnswer,
+        ) => {
+            clearTimeout(timers[i]);
+            timers[i] = setTimeout(() => settle(i, answer()), ms);
+            timers[i].unref?.();
+        };
+        // No `timedOut`: a call that never started never reached a handler.
+        batch.forEach((_, i) =>
+            arm(i, waits[i] + BATCH_START_GRACE_MS, () => ({
                 status: null,
-                error: err instanceof Error ? err.message : String(err),
+                error: 'dispatcher: call never started',
+            })),
+        );
+        const started = (i: number): void => {
+            if (!answered[i])
+                arm(i, batch[i].call.timeoutMs, () =>
+                    failedAnswer(timeoutError()),
+                );
+        };
+        const longest = Math.max(
+            ...batch.map(
+                ({ call }, i) =>
+                    waits[i] + BATCH_START_GRACE_MS + call.timeoutMs,
+            ),
+        );
+
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_BATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                    },
+                    body,
+                    signal: AbortSignal.any([
+                        hangUp.signal,
+                        AbortSignal.timeout(longest),
+                    ]),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
+
+            const dispatchError = response.headers.get(
+                EVENTS_DISPATCH_ERROR_HEADER,
+            );
+            if (dispatchError === 'no-such-route') {
+                // A dispatcher from before the batch route: nothing ran, so
+                // every call goes again on its own.
+                void response.body?.cancel().catch(() => {});
+                this.#batchUnsupportedUntil =
+                    Date.now() + BATCH_UNSUPPORTED_RETRY_MS;
+                batch.forEach(({ call }, i) => {
+                    if (!answered[i])
+                        void this.#sendOne(call).then((a) => settle(i, a));
+                });
+                return;
+            }
+            if (dispatchError !== null || !response.ok) {
+                void response.body?.cancel().catch(() => {});
+                // An unmarked failure (an edge 404, a WAF 403) reads as
+                // Puter's, never as any handler's answer.
+                settleRest(
+                    dispatchError !== null
+                        ? dispatchAnswer(dispatchError, response.status)
+                        : {
+                              status: null,
+                              error: `dispatcher: unmarked ${response.status} for a batch`,
+                          },
+                );
+                return;
+            }
+
+            let buffered = '';
+            const read = (line: string): void => {
+                if (line.trim() === '') return;
+                const parsed = parseAnswerLine(line);
+                if (!parsed || parsed.i >= batch.length) return;
+                if ('started' in parsed) started(parsed.i);
+                else settle(parsed.i, parsed.answer);
             };
+            const decoder = new TextDecoder();
+            for await (const chunk of response.body ?? []) {
+                buffered += decoder.decode(chunk, { stream: true });
+                let newline = buffered.indexOf('\n');
+                while (newline !== -1) {
+                    read(buffered.slice(0, newline));
+                    buffered = buffered.slice(newline + 1);
+                    newline = buffered.indexOf('\n');
+                }
+            }
+            read(buffered);
+            settleRest({
+                status: null,
+                error: 'dispatcher: batch ended without an answer',
+            });
+        } catch (err) {
+            // Every call has its answer and the request was dropped on purpose.
+            if (open === 0) return;
+            settleRest(failedAnswer(err));
+        }
+    }
+
+    /** Whether a call on `lane` may take a connection now. */
+    #free(lane: InvokeLane): boolean {
+        return (
+            this.#active < INVOKE_CONNECTIONS &&
+            (lane !== 'schedule' ||
+                this.#scheduleActive < SCHEDULE_INVOKE_CONNECTIONS)
+        );
+    }
+
+    #take(lane: InvokeLane): void {
+        this.#active++;
+        if (lane === 'schedule') this.#scheduleActive++;
+    }
+
+    /** A connection slot, or false once `waitMs` passes without one. */
+    #acquire(waitMs: number, lane: InvokeLane): Promise<boolean> {
+        if (this.#free(lane)) {
+            this.#take(lane);
+            return Promise.resolve(true);
+        }
+        return new Promise((resolve) => {
+            const granted = () => {
+                clearTimeout(timer);
+                resolve(true);
+            };
+            const timer = setTimeout(() => {
+                this.#waiting.delete(granted);
+                resolve(false);
+            }, waitMs);
+            timer.unref?.();
+            this.#waiting.set(granted, lane);
+        });
+    }
+
+    /**
+     * Give the slot back, then hand one to the longest waiter it can go to: a
+     * schedule waiter at a full schedule share doesn't hold up a delivery.
+     */
+    #release(lane: InvokeLane): void {
+        this.#active--;
+        if (lane === 'schedule') this.#scheduleActive--;
+        for (const [granted, waiting] of this.#waiting) {
+            if (!this.#free(waiting)) continue;
+            this.#waiting.delete(granted);
+            this.#take(waiting);
+            granted();
+            return;
         }
     }
 
@@ -373,18 +784,120 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
     }
 }
 
+/** What a failure the dispatcher marked as its own reads as. */
+const dispatchAnswer = (
+    reason: string,
+    status: number,
+): EventsInvokeAnswer => ({
+    status: null,
+    error: `dispatcher: ${reason} (${status})`,
+    dispatchReason: reason,
+});
+
+/** What `AbortSignal.timeout` rejects with, for a batched call's own timeout. */
+const timeoutError = (): Error =>
+    new DOMException(
+        'The operation was aborted due to timeout',
+        'TimeoutError',
+    );
+
+/** One line of a batch answer, or null when it is not one. */
+export const parseAnswerLine = (
+    line: string,
+):
+    | { i: number; started: true }
+    | { i: number; answer: EventsInvokeAnswer }
+    | null => {
+    let raw: unknown;
+    try {
+        raw = JSON.parse(line);
+    } catch {
+        return null;
+    }
+    if (!raw || typeof raw !== 'object') return null;
+    const { i, started, status, handled, error, retryAfter, dispatch } =
+        raw as Record<string, unknown>;
+    if (!Number.isInteger(i) || (i as number) < 0) return null;
+    if (started === true) return { i: i as number, started: true };
+    if (typeof status !== 'number') return null;
+    if (typeof dispatch === 'string')
+        return { i: i as number, answer: dispatchAnswer(dispatch, status) };
+    return {
+        i: i as number,
+        answer: answerFrom({
+            status,
+            headers: {
+                get: (name) => {
+                    if (name === EVENTS_HANDLED_HEADER)
+                        return handled === true ? '1' : null;
+                    if (name === EVENTS_ERROR_HEADER)
+                        return typeof error === 'string' ? error : null;
+                    if (name === 'retry-after')
+                        return typeof retryAfter === 'string'
+                            ? retryAfter
+                            : null;
+                    return null;
+                },
+            },
+        }),
+    };
+};
+
+/** Whether deploying the script is what fixes this answer. */
+const selfDeployable = (result: EventsInvokeAnswer): boolean =>
+    result.status === null
+        ? result.dispatchReason !== undefined &&
+          SELF_DEPLOYABLE_MISS_REASONS.has(result.dispatchReason)
+        : result.handled === true && result.error === BAD_KEY_ERROR;
+
+/** What a script's answer says, read the same way by every transport. */
+export const answerFrom = (response: {
+    status: number;
+    headers: { get(name: string): string | null };
+}): EventsInvokeAnswer => {
+    const errorHeader = response.headers.get(EVENTS_ERROR_HEADER);
+    const retryAfterMs = parseRetryAfterMs(response.headers.get('retry-after'));
+    return {
+        status: response.status,
+        handled: response.headers.get(EVENTS_HANDLED_HEADER) === '1',
+        ...(errorHeader ? { error: errorHeader } : {}),
+        ...(retryAfterMs ? { retryAfterMs } : {}),
+    };
+};
+
+/** An attempt that threw before any answer came back. */
+export const failedAnswer = (err: unknown): EventsInvokeAnswer => ({
+    status: null,
+    error: err instanceof Error ? err.message : String(err),
+    // What `AbortSignal.timeout` rejects with.
+    ...((err as { name?: unknown } | null)?.name === 'TimeoutError'
+        ? { timedOut: true }
+        : {}),
+});
+
+/** Whole seconds only; an HTTP-date form reads as absent. */
+const parseRetryAfterMs = (value: string | null): number => {
+    const seconds = Number(value);
+    return value && Number.isFinite(seconds) && seconds > 0
+        ? seconds * 1000
+        : 0;
+};
+
 /**
- * 2xx took it regardless. 429 is always "not now". Any other 4xx is a refusal
- * only when `handled` says the answer is provably the script's own — an
- * unmarked 4xx came from something between here and the handler.
+ * 2xx took it regardless. Anything else not marked as the script's own answer
+ * came from something between here and the handler. A marked answer is a
+ * refusal when it is a 4xx other than 429, Puter's failure when the runtime
+ * names one, and the handler's failure otherwise.
  */
 export const outcomeForStatus = (
     status: number,
     handled: boolean,
+    errorCode?: string,
 ): WorkerInvokeOutcome => {
     if (status >= 200 && status < 300) return 'settled';
+    if (!handled) return 'unavailable';
+    if (errorCode && PLATFORM_ERROR_CODES.has(errorCode)) return 'unavailable';
     if (status === 429) return 'retriable';
-    if (status >= 400 && status < 500)
-        return handled ? 'terminal' : 'retriable';
+    if (status >= 400 && status < 500) return 'terminal';
     return 'retriable';
 };

src/backend/controllers/auth/AuthController.test.ts

@@ -42,6 +42,10 @@ import {
     adminOnlyGate,
     requireUserActorGate,
 } from '../../core/http/middleware/gates.js';
+import {
+    configureRateLimit,
+    listConfiguredRateLimitBackends,
+} from '../../core/http/middleware/rateLimit.js';
 import {
     ROUTES_METADATA_KEY,
     type CollectedRoute,
@@ -1753,6 +1757,40 @@ describe('AuthController.handleLoginOtp + handleLoginRecoveryCode', () => {
         ).rejects.toMatchObject({ statusCode: 400 });
     });
 
+    it('refuses the sign-in while the used-token store is down', async () => {
+        const { user, secret, otpToken } = await startOtpLogin();
+        const token = await otpToken();
+
+        const { default: backend } = listConfiguredRateLimitBackends();
+        configureRateLimit({
+            default: 'redis',
+            redis: {
+                multi: () => {
+                    throw new Error('store down');
+                },
+            },
+        });
+        const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+        const res = makeRes();
+        try {
+            await expect(
+                controller.handleLoginOtp(
+                    makeReq({ token, code: liveTotp(user.username, secret) }),
+                    res,
+                ),
+            ).rejects.toThrow('store down');
+        } finally {
+            logged.mockRestore();
+            configureRateLimit({
+                default: backend,
+                redis: server.clients.redis,
+                kv: server.stores.kv,
+                metering: server.services.metering,
+            });
+        }
+        expect(res.body).toBeUndefined();
+    });
+
     it('a wrong code leaves the token usable for a retry', async () => {
         const { user, secret, otpToken } = await startOtpLogin();
         const token = await otpToken();
@@ -5094,6 +5132,26 @@ describe('AuthController phone verification — staging & reuse', () => {
         expect(staged).toBe('+14155550123');
     });
 
+    it('makes an email-gate-last account verify its card before its phone', async () => {
+        const { user, actor } = await makeUserAndActor({
+            requires_phone_verification: 1,
+            requires_card_verification: 1,
+        });
+        await server.stores.user.updateMetadata(user.id, {
+            emailGateLast: true,
+        });
+        const prelude = stubPrelude();
+        await withClients({ prelude }, async () => {
+            await expect(
+                controller.handleSendConfirmPhone(
+                    makeReq({ phone: '+14155550123' }, { actor }),
+                    makeRes(),
+                ),
+            ).rejects.toMatchObject({ statusCode: 409 });
+        });
+        expect(prelude.createVerification).not.toHaveBeenCalled();
+    });
+
     it('forwards an abuse-extension veto as 429 with the opaque reason, and sends nothing', async () => {
         const { user, actor } = await makeUserAndActor();
         const emitAndWait = vi.fn(
@@ -5159,7 +5217,7 @@ describe('AuthController phone verification — staging & reuse', () => {
         );
     });
 
-    it('carries the client dispatch id on the sent signal', async () => {
+    it('carries the client dispatch id on the check and sent events', async () => {
         const { actor } = await makeUserAndActor();
         const emit = vi.fn();
         const emitAndWait = vi.fn(async () => {});
@@ -5180,6 +5238,11 @@ describe('AuthController phone verification — staging & reuse', () => {
             expect.objectContaining({ dispatch_id: 'disp-123' }),
             expect.anything(),
         );
+        expect(emitAndWait).toHaveBeenCalledWith(
+            'puter.phone-verification.check',
+            expect.objectContaining({ dispatch_id: 'disp-123' }),
+            expect.anything(),
+        );
     });
 
     it('does NOT emit the sent signal when the send fails upstream', async () => {
@@ -7687,6 +7750,46 @@ describe('AuthController permission revokes', () => {
         );
         expect(res.body).toEqual({});
     });
+
+    it('revoke-user-user: refuses a key-value share grant and leaves it standing', async () => {
+        const { actor: ownerActor, user: owner } = await makeUserAndActor();
+        const { user: holder } = await makeUserAndActor();
+        const permissions = ['kv-share', 'kv-share-read', 'kv-share-write'].map(
+            (root) => `${root}:${owner.uuid}:os-global:notes:${uuidv4()}`,
+        );
+        for (const permission of permissions)
+            await server.stores.permission.upsertUserUserPerm(
+                holder.id,
+                owner.id,
+                permission,
+                {},
+            );
+
+        for (const permission of permissions)
+            await expect(
+                inCtx(ownerActor, () =>
+                    controller.handleRevokeUserUser(
+                        makeReq(
+                            { target_username: holder.username, permission },
+                            { actor: ownerActor },
+                        ),
+                        makeRes(),
+                    ),
+                ),
+            ).rejects.toMatchObject({
+                statusCode: 400,
+                message: expect.stringContaining('puter.kv.unshare()'),
+            });
+        expect(
+            (
+                await server.stores.permission.findUserUserPermsByIssuer(
+                    holder.id,
+                    owner.id,
+                    permissions,
+                )
+            ).sort(),
+        ).toEqual([...permissions].sort());
+    });
 });
 
 // ── Permission checks + listing ────────────────────────────────────

src/backend/controllers/auth/AuthController.ts

@@ -66,6 +66,7 @@ import {
     isCardFallbackEnabled,
     phoneAttemptsKey,
 } from '../../util/cardFallback.js';
+import { isEmailGateLast } from '../../util/verificationOrder.js';
 import { sessionCookieFlags } from '../../util/cookieFlags.js';
 import { cleanEmail, isStorableEmail } from '../../util/email.js';
 import { isGodmodeApp } from '../../util/godmodeApps.js';
@@ -1220,6 +1221,18 @@ export class AuthController extends PuterController {
         }
     }
 
+    /** The email-gate-last confirmation email; a failure never fails the gate. */
+    private async sendDeferredConfirmation(userId: number): Promise<void> {
+        try {
+            await this.services.signup.sendDeferredConfirmationEmail(userId);
+        } catch (e) {
+            console.warn(
+                '[verification] deferred confirmation email failed:',
+                e,
+            );
+        }
+    }
+
     private async isCardFallbackEligible(user: {
         id: number;
         requires_phone_verification?: boolean | number | null;
@@ -1265,6 +1278,12 @@ export class AuthController extends PuterController {
             throw new HttpError(403, 'Account suspended.', {
                 legacyCode: 'account_suspended',
             });
+        if (isEmailGateLast(user) && user.requires_card_verification)
+            throw new HttpError(
+                409,
+                'Card verification must be completed first.',
+                { legacyCode: 'conflict' },
+            );
         if (!this.clients.prelude?.isConfigured())
             throw await this.smsSendError(
                 503,
@@ -1335,6 +1354,7 @@ export class AuthController extends PuterController {
             user_uid: user.uuid,
             phone: parsed.e164,
             device_fingerprint: req.deviceFingerprint ?? null,
+            dispatch_id: dispatchId ?? null,
             allowed: true,
             reason: null as string | null,
             card_fallback_instead: false,
@@ -1573,6 +1593,7 @@ export class AuthController extends PuterController {
             requires_phone_verification: 0,
             phone: pendingPhone,
         });
+        await this.sendDeferredConfirmation(user.id);
 
         // Run the verified-event listeners synchronously (emitAndWait, not
         // fire-and-forget emit) so a carrier-based card-verification waiver in
@@ -1653,7 +1674,11 @@ export class AuthController extends PuterController {
             res.json({ card_verified: true });
             return;
         }
-        if (user.requires_phone_verification && !fallbackEligible)
+        if (
+            user.requires_phone_verification &&
+            !fallbackEligible &&
+            !isEmailGateLast(user)
+        )
             throw new HttpError(
                 409,
                 'Phone verification must be completed first.',
@@ -1716,6 +1741,7 @@ export class AuthController extends PuterController {
             await this.stores.user.update(user.id, {
                 requires_card_verification: 0,
             });
+            await this.sendDeferredConfirmation(user.id);
             res.json({ card_verified: true, disabled: true });
             return;
         }
@@ -1775,7 +1801,11 @@ export class AuthController extends PuterController {
             res.json({ card_verified: true });
             return;
         }
-        if (user.requires_phone_verification && !fallbackEligible)
+        if (
+            user.requires_phone_verification &&
+            !fallbackEligible &&
+            !isEmailGateLast(user)
+        )
             throw new HttpError(
                 409,
                 'Phone verification must be completed first.',
@@ -1816,6 +1846,7 @@ export class AuthController extends PuterController {
             await this.stores.user.update(user.id, {
                 requires_card_verification: 0,
             });
+            await this.sendDeferredConfirmation(user.id);
             res.json({ card_verified: true, disabled: true });
             return;
         }
@@ -1840,6 +1871,7 @@ export class AuthController extends PuterController {
             requires_card_verification: 0,
             ...(clearedPhoneGate ? { requires_phone_verification: 0 } : {}),
         });
+        await this.sendDeferredConfirmation(user.id);
 
         // Awaited like `user.phone-verified`, so whatever a listener grants on
         // verification is in place before the client refreshes. emitAndWait
@@ -3304,6 +3336,14 @@ export class AuthController extends PuterController {
                 { legacyCode: 'bad_request' },
             );
         }
+        // Only unsharing retires the handle along with its grant; withdrawing
+        // the grant alone would leave the share listed and dead.
+        if (typeof permission === 'string' && isKvSharePermission(permission))
+            throw new HttpError(
+                400,
+                'Stop a key-value share with `puter.kv.unshare()` (`DELETE /kv/shares/:handle`)',
+                { legacyCode: 'bad_request' },
+            );
         await this.services.permission.revokeUserUserPermission(
             req.actor!,
             target_username,
@@ -3810,8 +3850,9 @@ export class AuthController extends PuterController {
 
         let app = await this.stores.app.getByUid(app_uid);
         if (!app && resolvedFromOrigin) {
-            // Hosted-subdomain origins get the site owner stamped as the
-            // app's creator at bootstrap; external origins stay unowned.
+            // Hosted-subdomain and custom-domain origins get the site owner
+            // stamped as the app's creator at bootstrap; external origins
+            // stay unowned.
             // Canonical origin only, so alternate hosts share one row.
             const canonicalOrigin =
                 this.services.auth.canonicalizeOrigin(origin);
@@ -3827,9 +3868,10 @@ export class AuthController extends PuterController {
             // An origin's uid is a deterministic uuidv5, so a deleted app
             // reappears here under the identical uid. Withdraw any cross-app
             // data grants left pointing at it before this new row can inherit
-            // consent the user gave its predecessor. Only *this* path can reuse
-            // a uid: `AppStore.create` mints a random uuid4, which no deleted
-            // app can ever hold again.
+            // consent the user gave its predecessor. Only `createFromOrigin`
+            // reuses a uid, here or wherever else an origin app is bootstrapped:
+            // `AppStore.create` mints a random uuid4, which no deleted app can
+            // ever hold again.
             //
             // Called directly rather than through `app.changed`: the token is
             // issued below, so this has to be able to stop that, and
@@ -3840,6 +3882,9 @@ export class AuthController extends PuterController {
                 app_uid,
                 'uid reused by a new app',
             );
+            // Key-value shares the predecessor left read as gone already,
+            // being older than this row; this only cleans them up.
+            this.services.kvShare.retireSharesOlderThanApp(app);
         }
         if (!app) {
             throw new HttpError(404, `App ${app_uid} does not exist`, {
@@ -4636,13 +4681,19 @@ export class AuthController extends PuterController {
         await checkRateLimit(`${scope}:${userUid}`, limit, window);
     }
 
-    /** An `otp-login` JWT completes one sign-in; a second redemption is refused. */
+    /**
+     * An `otp-login` JWT completes one sign-in; a second redemption is refused.
+     * Fails closed: while the store can't say whether the token was used, the
+     * sign-in is refused rather than letting a captured token replay.
+     */
     async #consumeOtpLoginToken(token: string): Promise<void> {
         const digest = crypto.createHash('sha256').update(token).digest('hex');
         const firstUse = await checkRateLimit(
             `login-otp-token-used:${digest}`,
             1,
             OTP_LOGIN_TOKEN_USED_WINDOW_MS,
+            undefined,
+            { failOpen: false },
         );
         if (!firstUse) {
             throw new HttpError(400, 'Invalid token.', {

src/backend/controllers/broadcast/BroadcastController.ts

@@ -19,6 +19,7 @@
 
 import type { Request, Response } from 'express';
 import { Controller, Post } from '../../core/http/decorators.js';
+import { schedulesUnavailable } from '../../core/schedules.js';
 import type { BroadcastService } from '../../services/broadcast/BroadcastService.js';
 import type { ForwardBatch } from '../../services/events/forwardQueue.js';
 import { PuterController } from '../types.js';
@@ -108,6 +109,42 @@ export class BroadcastController extends PuterController {
         );
         res.status(200).json({ ok: true, ...reply });
     }
+
+    /**
+     * Schedule changes a region made for schedules this one owns, handed to
+     * `services.schedules`; `events_disabled` on a server without it.
+     */
+    @Post('/events/schedules', { subdomain: '*' })
+    async schedules(req: Request, res: Response): Promise<void> {
+        const broadcast = this.services.broadcast as unknown as
+            BroadcastService | undefined;
+        if (!broadcast) {
+            res.status(503).json({
+                error: { message: 'Broadcast service not registered' },
+            });
+            return;
+        }
+
+        const verified = await broadcast.verifySignedRequest(
+            req.rawBody,
+            broadcastHeaders(req),
+        );
+        if (!verified.ok) {
+            res.status(verified.status ?? 400).json({
+                error: { message: verified.message ?? 'Bad request' },
+            });
+            return;
+        }
+        if (verified.info?.ignored) {
+            res.status(200).json({ ok: true, ...verified.info });
+            return;
+        }
+
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        const reply = await schedules.receivePeerChanges(req.body);
+        res.status(200).json({ ok: true, ...reply });
+    }
 }
 
 const broadcastHeaders = (req: Request) => {

src/backend/controllers/drivers/DriverController.test.ts

@@ -736,6 +736,42 @@ describe('DriverController driver-method lifecycle events', () => {
             rejectReason: 'blocked in test',
         });
     });
+
+    it('answers 429 when a before listener vetoes with rejectStatus 429', async () => {
+        const actor = await makeUserActor();
+        await grantKvPerm(actor);
+        const key = `throttle-${uuidv4()}`;
+        server.clients.event.on(
+            'driver.puter-kvstore.set.before',
+            (_k, data) => {
+                if ((data.args as { key?: string })?.key === key) {
+                    data.allow = false;
+                    data.rejectStatus = 429;
+                }
+            },
+        );
+
+        const req = makeReq(
+            {
+                interface: 'puter-kvstore',
+                method: 'set',
+                args: { key, value: 'v' },
+            },
+            actor,
+        );
+        await expect(
+            runWithContext({ actor }, () =>
+                routes['POST /call'](
+                    req,
+                    makeRes() as unknown as Response,
+                    () => {},
+                ),
+            ),
+        ).rejects.toMatchObject({
+            statusCode: 429,
+            legacyCode: 'too_many_requests',
+        });
+    });
 });
 
 describe('DriverController.#handleListInterfaces', () => {

src/backend/controllers/drivers/DriverController.ts

@@ -31,6 +31,7 @@ import {
     checkDriverRateLimit,
 } from '../../core/http/middleware/rateLimit.js';
 import type { PuterRouter } from '../../core/http/PuterRouter.js';
+import type { LifecycleRejectStatus } from '../../clients/event/types.js';
 import type { DriverMeta } from '../../drivers/meta.js';
 import {
     isCreditExhaustion,
@@ -334,7 +335,8 @@ export class DriverController extends PuterController {
 
         if (req.actor) {
             const permService = this.services.permission as unknown as
-                PermissionService | undefined;
+                | PermissionService
+                | undefined;
             if (permService) {
                 // Build via PermissionUtil.join so any `:` in a driver or
                 // interface name is escaped — raw interpolation would let a
@@ -475,6 +477,7 @@ export class DriverController extends PuterController {
             args,
             allow: true as boolean,
             rejectReason: undefined as string | undefined,
+            rejectStatus: undefined as LifecycleRejectStatus | undefined,
         };
         await this.clients.event?.emitAndWait(
             `driver.${ifaceName}.${method}.before`,
@@ -496,11 +499,12 @@ export class DriverController extends PuterController {
                 },
                 {},
             );
+            const throttled = beforeEvent.rejectStatus === 429;
             throw new HttpError(
-                403,
+                throttled ? 429 : 403,
                 beforeEvent.rejectReason ??
                     `Blocked by policy: ${ifaceName}:${method}`,
-                { legacyCode: 'forbidden' },
+                { legacyCode: throttled ? 'too_many_requests' : 'forbidden' },
             );
         }
 

src/backend/controllers/events/EventsController.ts

@@ -25,6 +25,11 @@ import { bodyRecord } from '../../core/http/requestBody.js';
 import { DURABLE_LIST_LIMIT_CAP } from '../../stores/events/DurableSubscriptionStore.js';
 import { EVENTS_WORKERS_LIST_LIMIT_CAP } from '../../stores/events/EventHandlerStore.js';
 import { KV_HANDLE_LIST_LIMIT_CAP } from '../../stores/events/KvShareHandleStore.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    schedulesUnavailable,
+    type ScheduleServiceContract,
+} from '../../core/schedules.js';
 import { normalizeLimit } from '../../util/pagination.js';
 import { PuterController } from '../types.js';
 import { EVENTS_WORKER_PREFIX } from '../../services/events/workerRuntime.js';
@@ -104,6 +109,24 @@ export class EventsController extends PuterController {
         });
     }
 
+    /**
+     * POST /events/subscriptions/update — replace one subscription's `context`
+     * in place. An id the caller does not hold reads as absent.
+     */
+    @Post('/subscriptions/update', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async update(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        const { sub } = await this.services.events.updateDurable(
+            actor,
+            bodyRecord(req),
+        );
+        res.json(sub);
+    }
+
     /**
      * GET /events/fetch — what the caller missed, one page at a time.
      *
@@ -151,6 +174,88 @@ export class EventsController extends PuterController {
         res.json({});
     }
 
+    // -- Schedules ---------------------------------------------------
+    //
+    // An app's handler run on a timer, served by `services.schedules` when a
+    // server has it and answered `events_disabled` when it doesn't. Creating
+    // one needs an app context; listing, reading and removing are scoped the
+    // way the durable verbs are. The create and remove verbs spend the
+    // subscribe and unsubscribe budgets in the service, shared with those
+    // verbs.
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /** POST /events/schedules — create a schedule for the calling app. */
+    @Post('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async createSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().createFor(actor, bodyRecord(req)));
+    }
+
+    /** GET /events/schedules — the caller's schedules, one page at a time. */
+    @Get('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async listSchedules(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        const query = (req.query ?? {}) as Record<string, unknown>;
+
+        const page = await this.#schedules().list(actor, {
+            limit: normalizeLimit(query.limit, {
+                cap: SCHEDULE_LIST_LIMIT_CAP,
+            }),
+            cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
+            includeTotal: query.includeTotal === 'true',
+        });
+
+        res.json({
+            items: page.items,
+            ...(page.cursor ? { cursor: page.cursor } : {}),
+            ...(page.total !== undefined ? { total: page.total } : {}),
+        });
+    }
+
+    /**
+     * POST /events/schedules/remove — end one schedule. An id the caller does
+     * not hold reads as absent.
+     */
+    @Post('/schedules/remove', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async removeSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        await this.#schedules().remove(actor, bodyRecord(req).scheduleId);
+        res.json({});
+    }
+
+    /**
+     * GET /events/schedules/:scheduleId — one schedule, ended or not. An id the
+     * caller does not hold reads as absent.
+     */
+    @Get('/schedules/:scheduleId', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async getSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().get(actor, req.params.scheduleId));
+    }
+
     // -- Cross-user key-value handles --------------------------------
 
     /**
@@ -407,10 +512,14 @@ export class EventsController extends PuterController {
             res.json({ deployed: true });
             return;
         }
-        // `failed` and `throttled` are worth another attempt: the rest say
-        // this script is not what the app's handlers currently are.
+        // `failed`, `throttled` and `busy` are worth another attempt: the rest
+        // say this script is not what the app's handlers currently are.
         res.status(
-            outcome === 'failed' || outcome === 'throttled' ? 502 : 404,
+            outcome === 'failed' ||
+                outcome === 'throttled' ||
+                outcome === 'busy'
+                ? 502
+                : 404,
         ).json({
             deployed: false,
             reason: outcome,
@@ -437,8 +546,8 @@ export class EventsController extends PuterController {
             );
             return;
         }
-        this.services.events.useWorkerMissHandler((appUid, script) =>
-            deployer.ensure(appUid, script),
+        this.services.events.useWorkerMissHandler((appUid, script, options) =>
+            deployer.ensure(appUid, script, options),
         );
     }
 

src/backend/controllers/events/limits.test.ts

@@ -41,16 +41,30 @@ import {
     EVENTS_SINGLE_DELIVERY_LIMIT,
     EVENTS_WORKER_INVOCATION_LIMIT,
     limitFor,
+    tiered,
     type TieredLimit,
 } from './limits.js';
 
 const tiers: Array<[string, TieredLimit]> = [
-    ['durable subscriptions per account', EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER],
+    [
+        'durable subscriptions per account',
+        EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
+    ],
     ['durable subscriptions per app', EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP],
     ['live share handles per account', EVENTS_KV_HANDLES_PER_USER],
     ['live share handles per app', EVENTS_KV_HANDLES_PER_APP],
 ];
 
+describe('tiered', () => {
+    it('gives paid plans the base, and the free and temporary tiers their own', () => {
+        const limit = tiered(30, 20, 10);
+        expect(limitFor(limit, null)).toBe(30);
+        expect(limitFor(limit, 'some_plan')).toBe(30);
+        expect(limitFor(limit, DEFAULT_FREE_SUBSCRIPTION)).toBe(20);
+        expect(limitFor(limit, DEFAULT_TEMP_SUBSCRIPTION)).toBe(10);
+    });
+});
+
 describe('the tiered subscription quotas', () => {
     it.each(tiers)('%s never lets a free tier exceed paid', (_name, tier) => {
         for (const n of Object.values(tier.bySubscription))
@@ -105,9 +119,9 @@ describe('the tiered subscription quotas', () => {
     });
 
     it('holds an unrecognised plan to the paid base', () => {
-        expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan')).toBe(
-            EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
-        );
+        expect(
+            limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan'),
+        ).toBe(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit);
         expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, null)).toBe(
             EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
         );
@@ -131,9 +145,9 @@ describe('the delivery budgets', () => {
             expect(
                 limitFor(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT, plan),
             ).toBe(matched);
-            expect(
-                limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan),
-            ).toBe(evaluated);
+            expect(limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan)).toBe(
+                evaluated,
+            );
             expect(evaluated).toBeGreaterThanOrEqual(matched);
         }
     });

src/backend/controllers/events/limits.ts

@@ -53,7 +53,12 @@ export interface TieredLimit {
     bySubscription: Record<string, number>;
 }
 
-const tiered = (paid: number, free: number, temp: number): TieredLimit => ({
+/** Paid, free and temporary caps as a {@link TieredLimit}. Called by extensions. */
+export const tiered = (
+    paid: number,
+    free: number,
+    temp: number,
+): TieredLimit => ({
     limit: paid,
     bySubscription: {
         [DEFAULT_FREE_SUBSCRIPTION]: free,
@@ -93,6 +98,18 @@ export interface TieredQuota {
  */
 export const EVENTS_SESSION_SUBSCRIPTIONS_PER_SOCKET = 50;
 
+/**
+ * Live subscriptions one user may hold through a share handle someone else
+ * minted for them, across all of their connections.
+ *
+ * Those rows sit in the owner's keyspace, where each one costs a filter
+ * evaluation on the owner's writes. The count also takes in connections that
+ * dropped and haven't been reaped yet, so the cap sits far above what
+ * reconnects leave behind. The owner's own rows through the handle don't
+ * count.
+ */
+export const EVENTS_SESSION_SUBSCRIPTIONS_PER_KV_HANDLE = 1000;
+
 /**
  * Durable subscriptions one account may hold, across every app.
  *
@@ -125,6 +142,13 @@ export const EVENTS_DURABLE_SUBSCRIPTIONS_MAX =
  */
 export const SUSPENDED_ROW_TTL_DAYS = 30;
 
+/**
+ * Subscriptions stopped by a revoked grant that one account keeps for `list`.
+ * They hold no quota slot, so past this the oldest are deleted when the account
+ * subscribes again, keeping its rows bounded.
+ */
+export const EVENTS_REVOKED_SUBSCRIPTIONS_KEPT = 500;
+
 /**
  * Subscribe calls per minute, per user.
  *
@@ -204,17 +228,35 @@ export const EVENTS_KV_HANDLES_PER_APP = tiered(10_000, 2_000, 0);
 export const EVENTS_KV_HANDLES_PER_HOLDER = 256;
 
 /**
- * Recipients one account may share with, per minute and per hour. A batch
- * spends one per recipient, on top of the per-call mint budget.
+ * Recipients one account may share with, per minute and per hour. A call spends
+ * one per recipient that doesn't already hold the share, on top of the per-call
+ * mint budget.
  */
 export const KV_SHARE_GRANT_LIMITS = [
     userWindow('kvShare:grants:minute', 300),
     userWindow('kvShare:grants:hour', 2_000, 60 * 60_000),
 ] as const;
 
+/**
+ * Recipients one account may name in share calls per minute, existing shares
+ * included. Each one is looked up before the call knows whether it changes
+ * anything, so a client that shares again on every load is bounded here.
+ */
+export const KV_SHARE_LOOKUP_LIMIT = userWindow('kvShare:lookups', 1_200);
+
 /** Recipients one `POST /kv/shares` call may name. */
 export const KV_SHARE_BATCH_MAX = 50;
 
+/**
+ * Notifications one party of a share may send the other through share handles,
+ * per minute and per day, in one app. Keeps one share from spending the
+ * recipient's whole notification allowance.
+ */
+export const KV_SHARE_NOTIFY_LIMITS = [
+    userWindow('kvShare:notify:minute', 30),
+    userWindow('kvShare:notify:day', 500, 24 * 60 * 60_000),
+] as const;
+
 // -- Handler surface -------------------------------------------------
 
 /**
@@ -311,8 +353,11 @@ export const EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT = tiered(512, 128, 128);
  */
 export const EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT = tiered(2048, 512, 512);
 
-/** Omit all inline KV values when the matched audience exceeds this count. */
-export const EVENTS_KV_VALUE_OMIT_MATCHED_SUBSCRIPTIONS = 128;
+/**
+ * Deliveries of one key-value change that carry its value inline, in the order
+ * the fan-out serves them. Later ones carry the key alone.
+ */
+export const EVENTS_KV_VALUE_DELIVERIES_PER_EVENT = 128;
 
 /**
  * Largest inline KV value in serialized bytes. Larger values are omitted;
@@ -404,16 +449,20 @@ export const EVENTS_REGION_PENDING_CEILING = 1_000_000;
 // a handler that answers "no" is not. Retrying on a fixed cadence turns one
 // broken deploy into a permanent load on whatever is failing, so the wait
 // doubles per attempt up to a ceiling, and a run of failures stops the
-// subscription rather than retrying it forever.
+// subscription rather than retrying it forever. A run Puter could not start or
+// finish backs off the same way but is not the handler's failure, so it never
+// counts toward that run.
 
-/** Wait before the first retry of a delivery a handler could not take. */
-export const EVENTS_RETRY_BASE_MS = 2_000;
-
-/** Longest wait between retries, however many have failed. */
-export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
+// Wait before the first retry of a delivery a handler could not take, the
+// longest wait, and the doubling between them. Shared with system jobs.
+export {
+    deliveryBackoffMs,
+    EVENTS_RETRY_BASE_MS,
+    EVENTS_RETRY_MAX_MS,
+} from '../../util/retryBackoff.js';
 
 /**
- * Failures in a row before a subscription is suspended. Counted per
+ * Handler failures in a row before a subscription is suspended. Counted per
  * subscription and reset by the first delivery a handler takes, so an
  * occasional failure never accumulates into one.
  */
@@ -426,13 +475,6 @@ export const EVENTS_CONSECUTIVE_FAILURES = 5;
  */
 export const EVENTS_FAILURE_COUNTER_TTL_MS = 60 * 60 * 1000;
 
-/** How long to hold a delivery whose handler has failed `attempts` times. */
-export const deliveryBackoffMs = (attempts: number): number =>
-    Math.min(
-        EVENTS_RETRY_MAX_MS,
-        EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
-    );
-
 // -- Suspended backlog -----------------------------------------------
 //
 // A suspended subscription stops metering but keeps what it is owed, and that

src/backend/controllers/events/scheduleRoutes.test.ts

@@ -0,0 +1,274 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The schedule routes are thin delegates: without `services.schedules` every
+ * one answers 503 `events_disabled`; with it, each hands its request over and
+ * answers what it returns.
+ */
+
+import type { Request, Response } from 'express';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    ScheduleServiceContract,
+    ScheduleView,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { BroadcastController } from '../broadcast/BroadcastController.js';
+import type { EventsController } from './EventsController.js';
+
+let server: PuterServer;
+let events: EventsController;
+let broadcast: BroadcastController;
+let actor: Actor;
+let saved: unknown;
+
+const SCHEDULE_ID = '0b3a3c9e-9f61-4f0e-8d2a-2f1b6f8f2c11';
+
+const view = {
+    scheduleId: SCHEDULE_ID,
+    kind: 'recurring',
+    appUid: 'app-x',
+} as ScheduleView;
+
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+
+const fakeSchedules = () =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(async () => view),
+        list: vi.fn(async () => ({
+            items: [view],
+            cursor: 'next',
+            total: 3,
+        })),
+        get: vi.fn(async () => view),
+        remove: vi.fn(async () => {}),
+        receivePeerChanges: vi.fn(async () => ({ applied: 2 })),
+        prepareSendAt: vi.fn(),
+        createDeferredAppMessage: vi.fn(),
+    }) satisfies ScheduleServiceContract;
+
+const req = (init: {
+    body?: unknown;
+    query?: Record<string, unknown>;
+    params?: Record<string, string>;
+}): Request =>
+    ({
+        actor,
+        body: init.body ?? {},
+        query: init.query ?? {},
+        params: init.params ?? {},
+        headers: {},
+        rawBody: Buffer.from(JSON.stringify(init.body ?? {})),
+    }) as unknown as Request;
+
+const res = () => {
+    const sent: { status: number; body: unknown } = {
+        status: 200,
+        body: undefined,
+    };
+    const response = {
+        json: vi.fn((body: unknown) => {
+            sent.body = body;
+            return response;
+        }),
+        status: vi.fn((code: number) => {
+            sent.status = code;
+            return response;
+        }),
+        setHeader: vi.fn(() => response),
+    };
+    return { response: response as unknown as Response, sent };
+};
+
+/** Each route, called as the router would. */
+const routes: Array<[string, (response: Response) => Promise<void>]> = [
+    [
+        'POST /events/schedules',
+        (r) => events.createSchedule(req({ body: { every: 60_000 } }), r),
+    ],
+    [
+        'GET /events/schedules',
+        (r) => events.listSchedules(req({ query: { limit: '5' } }), r),
+    ],
+    [
+        'GET /events/schedules/:scheduleId',
+        (r) =>
+            events.getSchedule(req({ params: { scheduleId: SCHEDULE_ID } }), r),
+    ],
+    [
+        'POST /events/schedules/remove',
+        (r) =>
+            events.removeSchedule(
+                req({ body: { scheduleId: SCHEDULE_ID } }),
+                r,
+            ),
+    ],
+    [
+        'POST /broadcast/events/schedules',
+        (r) =>
+            broadcast.schedules(
+                req({ body: { namespace: 'n', changes: [] } }),
+                r,
+            ),
+    ],
+];
+
+beforeAll(async () => {
+    server = await setupTestServer();
+    events = server.controllers.events as unknown as EventsController;
+    broadcast = server.controllers.broadcast as unknown as BroadcastController;
+    actor = makeActor({
+        user: { id: 1, uuid: 'u-1', username: 'someone' },
+    } as never);
+    saved = slot().schedules;
+    vi.spyOn(
+        server.services.broadcast,
+        'verifySignedRequest',
+    ).mockResolvedValue({ ok: true, peerId: 'peer' } as never);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+});
+
+afterAll(async () => {
+    vi.restoreAllMocks();
+    await server?.shutdown();
+});
+
+describe('without services.schedules', () => {
+    it.each(routes)('%s answers 503 events_disabled', async (_name, call) => {
+        setSchedules(undefined);
+        const { response } = res();
+        let thrown: unknown;
+        try {
+            await call(response);
+        } catch (err) {
+            thrown = err;
+        }
+        expect(isHttpError(thrown)).toBe(true);
+        expect(thrown).toMatchObject({
+            statusCode: 503,
+            legacyCode: 'events_disabled',
+            // The designed answer, so it never pages.
+            noAlarm: true,
+        });
+    });
+
+    it('refuses an unsigned peer batch before saying schedules are off', async () => {
+        setSchedules(undefined);
+        vi.spyOn(
+            server.services.broadcast,
+            'verifySignedRequest',
+        ).mockResolvedValueOnce({
+            ok: false,
+            status: 401,
+            message: 'bad signature',
+        } as never);
+        const { response, sent } = res();
+        await broadcast.schedules(
+            req({ body: { namespace: 'n', changes: [] } }),
+            response,
+        );
+        expect(sent.status).toBe(401);
+    });
+});
+
+describe('with services.schedules', () => {
+    it('creates through createFor and answers its view', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.createSchedule(req({ body: { every: 60_000 } }), response);
+        expect(schedules.createFor).toHaveBeenCalledWith(actor, {
+            every: 60_000,
+        });
+        expect(sent.body).toEqual(view);
+    });
+
+    it('lists with the limit capped, the cursor and includeTotal', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.listSchedules(
+            req({
+                query: { limit: '5000', cursor: 'c', includeTotal: 'true' },
+            }),
+            response,
+        );
+        expect(schedules.list).toHaveBeenCalledWith(actor, {
+            limit: 200,
+            cursor: 'c',
+            includeTotal: true,
+        });
+        expect(sent.body).toEqual({ items: [view], cursor: 'next', total: 3 });
+    });
+
+    it('reads one through get', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.getSchedule(
+            req({ params: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.get).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual(view);
+    });
+
+    it('removes through remove and answers {}', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.removeSchedule(
+            req({ body: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.remove).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual({});
+    });
+
+    it('hands a signed peer batch to receivePeerChanges', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        const body = { namespace: 'n', changes: [] };
+        await broadcast.schedules(req({ body }), response);
+        expect(schedules.receivePeerChanges).toHaveBeenCalledWith(body);
+        expect(sent).toEqual({ status: 200, body: { ok: true, applied: 2 } });
+    });
+});

src/backend/controllers/events/workerDeploy.test.ts

@@ -25,8 +25,13 @@
 
 import MockRedis from 'ioredis-mock';
 import { describe, expect, it } from 'vitest';
+import type { EventsInvokeCall } from '../../clients/events/EventsWorkerInvokerClient.js';
 import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
-import { EventsWorkerDeployer } from './workerDeploy.js';
+import {
+    EventsWorkerDeployer,
+    LocalEventsInvokeTransport,
+    type EventsDeployOutcome,
+} from './workerDeploy.js';
 import { eventsWorkerScript } from '../../services/events/workerRuntime.js';
 import { handlerSetHash } from '../../services/events/workerSource.js';
 import type { IConfig } from '../../types.js';
@@ -49,6 +54,8 @@ type Layers = ConstructorParameters<typeof EventsWorkerDeployer>[0];
 const makeLayers = (owners: Record<string, Owner | null>) => {
     const redis = new MockRedis.Cluster(['redis://localhost:7001']);
     const created: string[] = [];
+    /** Apps whose handler sources were read, once per read. */
+    const sourceReads: string[] = [];
     const layers = {
         config: {
             events: { workerRuntime: true, internalSecret: 'sekret' },
@@ -60,16 +67,20 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
         clients: { redis },
         stores: {
             eventHandler: {
-                allForApp: async (appUid: string) => [
-                    {
-                        appUid,
-                        name: 'h',
-                        source: 'async () => {}',
-                        sourceHash: SOURCE_HASH,
-                        createdAt: 0,
-                        updatedAt: 0,
-                    },
-                ],
+                setForApp: async () => [{ name: 'h', sourceHash: SOURCE_HASH }],
+                allForApp: async (appUid: string) => {
+                    sourceReads.push(appUid);
+                    return [
+                        {
+                            appUid,
+                            name: 'h',
+                            source: 'async () => {}',
+                            sourceHash: SOURCE_HASH,
+                            createdAt: 0,
+                            updatedAt: 0,
+                        },
+                    ];
+                },
             },
             app: {
                 getByUid: async (appUid: string) => {
@@ -104,7 +115,12 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
             },
         },
     };
-    return { layers: layers as unknown as Layers, created, redis };
+    return {
+        layers: layers as unknown as Layers,
+        created,
+        redis,
+        sourceReads,
+    };
 };
 
 describe('EventsWorkerDeployer.ensure', () => {
@@ -127,8 +143,10 @@ describe('EventsWorkerDeployer.ensure', () => {
         expect(b).toBe('no-owner');
     });
 
-    it('refuses once the app has deployed past the hourly cap', async () => {
-        const { layers, redis } = makeLayers({ 'app-c': { ownerUserId: 3 } });
+    it('refuses once the app has deployed past the hourly cap, before reading or building any source', async () => {
+        const { layers, redis, sourceReads, created } = makeLayers({
+            'app-c': { ownerUserId: 3 },
+        });
         const deployer = new EventsWorkerDeployer(layers);
         const hour = new Date().toISOString().slice(0, 13);
         await redis.set(
@@ -137,6 +155,24 @@ describe('EventsWorkerDeployer.ensure', () => {
         );
 
         expect(await deployer.ensure('app-c', SCRIPT)).toBe('throttled');
+        expect(sourceReads).toEqual([]);
+        expect(created).toEqual([]);
+    });
+
+    it('answers a stale script without spending the hourly cap', async () => {
+        const { layers, redis, sourceReads } = makeLayers({
+            'app-e': { ownerUserId: 5 },
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+        const stale = eventsWorkerScript(
+            handlerSetHash([{ name: 'h', sourceHash: 'b'.repeat(64) }]),
+            '',
+        );
+
+        expect(await deployer.ensure('app-e', stale)).toBe('stale');
+        expect(sourceReads).toEqual([]);
+        const hour = new Date().toISOString().slice(0, 13);
+        expect(await redis.get(`ev:deploys:{app-e}:${hour}`)).toBeNull();
     });
 
     it('deploys normally under the cap', async () => {
@@ -145,3 +181,151 @@ describe('EventsWorkerDeployer.ensure', () => {
         expect(await deployer.ensure('app-d', SCRIPT)).toBe('deployed');
     });
 });
+
+/** A server's schedules with a deploy gate that records and can refuse. */
+const withGate = (
+    layers: Layers,
+    refuse: (capped: boolean) => boolean = () => false,
+) => {
+    const gated: Array<{ script: string; capped: boolean }> = [];
+    const gate = {
+        gateDeploy: async (
+            script: string,
+            options: { capped: boolean },
+            deploy: () => Promise<EventsDeployOutcome>,
+        ): Promise<EventsDeployOutcome> => {
+            gated.push({ script, capped: options.capped });
+            return refuse(options.capped) ? 'busy' : deploy();
+        },
+    };
+    const deployer = new EventsWorkerDeployer({
+        ...layers,
+        services: { ...layers.services, schedules: gate },
+    } as unknown as Layers);
+    return { deployer, gated };
+};
+
+describe('the schedules` deploy gate', () => {
+    it('is skipped without schedules, or with schedules that have no gate', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        await expect(
+            new EventsWorkerDeployer(layers).ensure('app-x', SCRIPT, {
+                lane: 'schedule',
+            }),
+        ).resolves.toBe('deployed');
+        const gateless = new EventsWorkerDeployer({
+            ...layers,
+            services: { ...layers.services, schedules: {} },
+        } as unknown as Layers);
+        await expect(
+            gateless.ensure('app-x', SCRIPT, { lane: 'schedule' }),
+        ).resolves.toBe('deployed');
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('takes every deploy, capped only for a schedule run', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers);
+
+        await expect(deployer.ensure('app-x', SCRIPT)).resolves.toBe(
+            'deployed',
+        );
+        await expect(
+            deployer.ensure('app-x', SCRIPT, { lane: 'schedule' }),
+        ).resolves.toBe('deployed');
+        expect(gated).toEqual([
+            { script: SCRIPT, capped: false },
+            { script: SCRIPT, capped: true },
+        ]);
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('answers a refused schedule run busy, and a delivery still deploys', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer } = withGate(layers, (capped) => capped);
+
+        await expect(
+            deployer.ensure('app-x', SCRIPT, { lane: 'schedule' }),
+        ).resolves.toBe('busy');
+        await expect(deployer.ensure('app-x', SCRIPT)).resolves.toBe(
+            'deployed',
+        );
+        expect(created).toEqual([SCRIPT]);
+    });
+
+    it('lets deliveries that join a schedule run`s refused deploy deploy once', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const scheduled = deployer.ensure('app-x', SCRIPT, {
+            lane: 'schedule',
+        });
+        const deliveries = [
+            deployer.ensure('app-x', SCRIPT),
+            deployer.ensure('app-x', SCRIPT),
+        ];
+        await expect(scheduled).resolves.toBe('busy');
+        await expect(Promise.all(deliveries)).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(created).toEqual([SCRIPT]);
+        expect(gated.map((g) => g.capped)).toEqual([true, false]);
+    });
+
+    it('lets a schedule run join a delivery`s deploy in flight', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const delivery = deployer.ensure('app-x', SCRIPT);
+        const scheduled = deployer.ensure('app-x', SCRIPT, {
+            lane: 'schedule',
+        });
+        await expect(Promise.all([delivery, scheduled])).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(gated).toEqual([{ script: SCRIPT, capped: false }]);
+        expect(created).toEqual([SCRIPT]);
+    });
+});
+
+describe('LocalEventsInvokeTransport', () => {
+    it('renders each request as it goes out, after a cold deploy', async () => {
+        const bodies: string[] = [];
+        let resident = false;
+        const transport = new LocalEventsInvokeTransport(
+            {
+                localworkerservice: {
+                    dispatchEventsWorker: async (request: Request) => {
+                        bodies.push(await request.text());
+                        if (!resident) return null;
+                        return new Response(null, {
+                            status: 200,
+                            headers: { 'x-puter-events-handled': '1' },
+                        });
+                    },
+                },
+            } as never,
+            {
+                ensure: async () => {
+                    resident = true;
+                    return 'deployed';
+                },
+            } as unknown as EventsWorkerDeployer,
+        );
+        let renders = 0;
+        const call: EventsInvokeCall = {
+            script: SCRIPT,
+            appUid: 'app-x',
+            key: 'k',
+            body: '{"n":0}',
+            timeoutMs: 5000,
+            render: () => `{"n":${++renders}}`,
+        };
+        await expect(transport.send(call)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(bodies).toEqual(['{"n":1}', '{"n":2}']);
+    });
+});

src/backend/controllers/events/workerDeploy.ts

@@ -26,18 +26,27 @@ import {
     INTERNAL_ADMISSION_BYPASS,
     INTERNAL_DEPLOY_TARGET,
 } from '../../drivers/workers/WorkerDriver.js';
-import type { EventsInvokeCall } from '../../clients/events/EventsWorkerInvokerClient.js';
+import type {
+    EventsInvokeAnswer,
+    EventsInvokeCall,
+    InvokeLane,
+} from '../../clients/events/EventsWorkerInvokerClient.js';
 import {
-    EVENTS_HANDLED_HEADER,
     EVENTS_INVOKE_PATH,
+    answerFrom,
+    failedAnswer,
 } from '../../clients/events/EventsWorkerInvokerClient.js';
 import type { puterServices } from '../../services/index.js';
+import type { IExtensionServiceInstances } from '../../services/types.js';
 import {
     eventsInvokeKey,
     eventsWorkerScope,
     eventsWorkerScript,
 } from '../../services/events/workerRuntime.js';
-import { generateEventsWorkerSource } from '../../services/events/workerSource.js';
+import {
+    generateEventsWorkerSource,
+    handlerSetHash,
+} from '../../services/events/workerSource.js';
 import type { puterStores } from '../../stores/index.js';
 import type { IConfig, LayerInstances } from '../../types.js';
 import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
@@ -54,13 +63,25 @@ import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
 
 /** Why a deploy could not happen, for the callers that answer differently. */
 export type EventsDeployOutcome =
-    'deployed' | 'stale' | 'no-handlers' | 'no-owner' | 'throttled' | 'failed';
+    | 'deployed'
+    | 'stale'
+    | 'no-handlers'
+    | 'no-owner'
+    | 'throttled'
+    | 'failed'
+    | 'busy';
+
+/** Whose call asked for the deploy: a schedule run's is held to a share. */
+export interface EventsDeployOptions {
+    lane?: InvokeLane;
+}
 
 interface DeployLayers {
     config: IConfig;
     clients: LayerInstances<typeof puterClients>;
     stores: LayerInstances<typeof puterStores>;
-    services: LayerInstances<typeof puterServices>;
+    services: LayerInstances<typeof puterServices> &
+        Pick<IExtensionServiceInstances, 'schedules'>;
     drivers: LayerInstances<typeof puterDrivers>;
 }
 
@@ -70,6 +91,12 @@ interface DeployLayers {
  */
 const DEPLOY_THROTTLE_TTL_SECONDS = 2 * 60 * 60;
 
+interface InFlightDeploy {
+    /** A schedule run's, which the region's deploy slots may refuse. */
+    capped: boolean;
+    promise: Promise<EventsDeployOutcome>;
+}
+
 interface DeployResult {
     success?: boolean;
     errors?: unknown[];
@@ -78,7 +105,7 @@ interface DeployResult {
 export class EventsWorkerDeployer {
     readonly #layers: DeployLayers;
     /** Collapses the burst of misses one cold app produces into one deploy. */
-    readonly #inFlight = new Map<string, Promise<EventsDeployOutcome>>();
+    readonly #inFlight = new Map<string, InFlightDeploy>();
 
     constructor(layers: DeployLayers) {
         this.#layers = layers;
@@ -97,15 +124,79 @@ export class EventsWorkerDeployer {
      * A request for anything else is `stale` — the set moved on, and the next
      * delivery resolves the new script rather than this one being revived.
      */
-    ensure(appUid: string, script: string): Promise<EventsDeployOutcome> {
+    ensure(
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions = {},
+    ): Promise<EventsDeployOutcome> {
         const key = `${appUid}|${script}`;
+        const capped = options.lane === 'schedule';
         const existing = this.#inFlight.get(key);
-        if (existing) return existing;
-        const started = this.#deploy(appUid, script).finally(() =>
-            this.#inFlight.delete(key),
+        if (!existing) return this.#start(key, appUid, script, options);
+        if (capped || !existing.capped) return existing.promise;
+        return this.#afterCapped(
+            existing.promise,
+            key,
+            appUid,
+            script,
+            options,
+        );
+    }
+
+    #start(
+        key: string,
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const entry: InFlightDeploy = {
+            capped: options.lane === 'schedule',
+            promise: this.#gated(appUid, script, options).finally(() => {
+                if (this.#inFlight.get(key) === entry)
+                    this.#inFlight.delete(key);
+            }),
+        };
+        this.#inFlight.set(key, entry);
+        return entry.promise;
+    }
+
+    /**
+     * A delivery that joined a schedule run's deploy: `busy` means no deploy
+     * started, and a delivery is never refused a slot, so it asks again.
+     */
+    async #afterCapped(
+        capped: Promise<EventsDeployOutcome>,
+        key: string,
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const outcome = await capped;
+        if (outcome !== 'busy') return outcome;
+        const current = this.#inFlight.get(key);
+        if (current && !current.capped) return current.promise;
+        return this.#start(key, appUid, script, options);
+    }
+
+    /**
+     * Every deploy goes through the schedules' deploy gate when a server has
+     * one: a per-script lock, so a burst of runs of an evicted script deploys
+     * it once, and a region-wide cap that a schedule run's deploy (`capped`)
+     * may be refused with `busy`. A delivery's deploy is never refused.
+     */
+    #gated(
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const deploy = () => this.#deploy(appUid, script);
+        return (
+            this.#layers.services.schedules?.gateDeploy?.(
+                script,
+                { capped: options.lane === 'schedule' },
+                deploy,
+            ) ?? deploy()
         );
-        this.#inFlight.set(key, started);
-        return started;
     }
 
     async #deploy(
@@ -113,20 +204,24 @@ export class EventsWorkerDeployer {
         script: string,
     ): Promise<EventsDeployOutcome> {
         const { stores, drivers } = this.#layers;
+        const scope = eventsWorkerScope(this.#layers.config);
 
-        const handlers = await stores.eventHandler.allForApp(appUid);
-        if (handlers.length === 0) return 'no-handlers';
-
-        const generated = generateEventsWorkerSource(handlers);
+        // Names and hashes are enough to tell a stale request, so the set's
+        // sources are read and built only once the throttle has let it through.
+        const set = await stores.eventHandler.setForApp(appUid);
+        if (set.length === 0) return 'no-handlers';
         // Two apps that published byte-identical sets share one script, which
         // is sound: the source is the same, and everything per-app arrives with
         // the invocation. Whoever deploys first owns it.
-        if (
-            eventsWorkerScript(
-                generated.setHash,
-                eventsWorkerScope(this.#layers.config),
-            ) !== script
-        )
+        if (eventsWorkerScript(handlerSetHash(set), scope) !== script)
+            return 'stale';
+
+        if (await this.#tooManyDeploysThisHour(appUid)) return 'throttled';
+
+        const handlers = await stores.eventHandler.allForApp(appUid);
+        const generated = generateEventsWorkerSource(handlers);
+        // The set can move between the two reads.
+        if (eventsWorkerScript(generated.setHash, scope) !== script)
             return 'stale';
         if (generated.tooLarge) {
             console.error(
@@ -140,8 +235,6 @@ export class EventsWorkerDeployer {
                 generated.broken.join(', '),
             );
 
-        if (await this.#tooManyDeploysThisHour(appUid)) return 'throttled';
-
         const owner = await this.#owner(appUid);
         if (!owner) return 'no-owner';
 
@@ -289,11 +382,9 @@ export class LocalEventsInvokeTransport {
         this.#deployer = deployer;
     }
 
-    async send(call: EventsInvokeCall): Promise<{
-        status: number | null;
-        handled?: boolean;
-        error?: string;
-    }> {
+    async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        // Built per attempt, so a send time stamped into the body is taken
+        // after a cold deploy rather than before it.
         const request = (): Request =>
             new Request(
                 `http://${call.script}.workers.puter.localhost${EVENTS_INVOKE_PATH}`,
@@ -303,7 +394,7 @@ export class LocalEventsInvokeTransport {
                         'content-type': 'application/json',
                         'x-puter-events-key': call.key,
                     },
-                    body: call.body,
+                    body: call.render ? call.render() : call.body,
                 },
             );
 
@@ -314,19 +405,21 @@ export class LocalEventsInvokeTransport {
             const outcome = await this.#deployer.ensure(
                 call.appUid,
                 call.script,
+                { lane: call.lane },
             );
             if (outcome !== 'deployed')
-                return { status: null, error: `deploy: ${outcome}` };
+                return {
+                    status: null,
+                    error: `deploy: ${outcome}`,
+                    ...(outcome === 'busy' ? { deployBusy: true } : {}),
+                };
 
             const retried = await this.#dispatch(request(), call.timeoutMs);
             if (retried === null)
                 return { status: null, error: 'script not resident' };
             return retried;
         } catch (err) {
-            return {
-                status: null,
-                error: err instanceof Error ? err.message : String(err),
-            };
+            return failedAnswer(err);
         }
     }
 
@@ -337,7 +430,7 @@ export class LocalEventsInvokeTransport {
     async #dispatch(
         request: Request,
         timeoutMs: number,
-    ): Promise<{ status: number; handled: boolean } | null> {
+    ): Promise<EventsInvokeAnswer | null> {
         let timer: ReturnType<typeof setTimeout> | undefined;
         const timeout = new Promise<'timeout'>((resolve) => {
             timer = setTimeout(() => resolve('timeout'), timeoutMs);
@@ -348,12 +441,10 @@ export class LocalEventsInvokeTransport {
                 this.#services.localworkerservice.dispatchEventsWorker(request),
                 timeout,
             ]);
-            if (response === 'timeout') throw new Error('invocation timed out');
+            if (response === 'timeout')
+                throw new DOMException('invocation timed out', 'TimeoutError');
             if (response === null) return null;
-            return {
-                status: response.status,
-                handled: response.headers.get(EVENTS_HANDLED_HEADER) === '1',
-            };
+            return answerFrom(response);
         } finally {
             clearTimeout(timer);
         }

src/backend/controllers/kvShare/KvShareController.http.test.ts

@@ -42,6 +42,7 @@ import {
     EVENTS_KV_HANDLES_PER_USER,
     KV_SHARE_BATCH_MAX,
     KV_SHARE_GRANT_LIMITS,
+    KV_SHARE_LOOKUP_LIMIT,
 } from '../events/limits.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
@@ -67,6 +68,7 @@ const resetBudgets = async (env: PuterTestEnv) => {
         const user = await env.server.stores.user.getByUsername(name);
         for (const scope of [
             EVENTS_KV_HANDLE_LIMIT.scope,
+            KV_SHARE_LOOKUP_LIMIT.scope,
             ...KV_SHARE_GRANT_LIMITS.map((window) => window.scope),
         ])
             await env.server.clients.redis.del(`rate:${scope}:${user!.id}`);
@@ -123,6 +125,7 @@ describe('/kv/shares', () => {
                 'handle',
                 'holder',
                 'label',
+                'notify',
                 'prefix',
             ].sort(),
         );
@@ -131,6 +134,42 @@ describe('/kv/shares', () => {
             appUid: null,
             holder: env.users.other.username,
             label: 'Groceries',
+            notify: false,
+        });
+    });
+
+    it('takes notify, keeps it on a re-share that leaves it out, and shows it to both sides', async () => {
+        const prefix = `notify-${uuidv4().slice(0, 8)}:`;
+        const on = await shareWithOther({ prefix, notify: true });
+        expect(on).toMatchObject({ prefix, notify: true });
+
+        const kept = await shareWithOther({ prefix, access: 'write' });
+        expect(kept).toMatchObject({ handle: on.handle, notify: true });
+
+        const listed = async (path: string, token: string) =>
+            (
+                (await (await req('GET', path, token)).json()) as {
+                    items: Array<Record<string, unknown>>;
+                }
+            ).items.find((row) => row.handle === on.handle);
+        expect(
+            await listed('/kv/shares?limit=200', env.users.user.token),
+        ).toMatchObject({ notify: true });
+        expect(
+            await listed('/kv/shares/with-me?limit=200', env.users.other.token),
+        ).toMatchObject({ notify: true });
+
+        const off = await shareWithOther({ prefix, notify: false });
+        expect(off).toMatchObject({ handle: on.handle, notify: false });
+
+        const refused = await req('POST', '/kv/shares', env.users.user.token, {
+            prefix,
+            recipient: env.users.other.username,
+            notify: 'yes',
+        });
+        expect(refused.status).toBe(400);
+        await expect(refused.json()).resolves.toMatchObject({
+            code: 'kv_share_notify_invalid',
         });
     });
 
@@ -169,6 +208,7 @@ describe('/kv/shares', () => {
             appUid: null,
             holder: env.users.other.username,
             label: 'Trip',
+            notify: false,
             createdAt: shared.createdAt,
             revokedAt: null,
             revokedBy: null,
@@ -190,6 +230,7 @@ describe('/kv/shares', () => {
             access: 'write',
             appUid: null,
             label: 'Trip',
+            notify: false,
             createdAt: shared.createdAt,
         });
         expect(withMeText).not.toContain(env.users.user.username);
@@ -622,6 +663,162 @@ describe('/kv/shares', () => {
             }
         });
     });
+
+    describe('sharing again', () => {
+        let ownerId: number;
+        let third: string;
+
+        beforeAll(async () => {
+            third = `kv-again-${uuidv4().slice(0, 8)}`;
+            await createTestUser(env.server, {
+                username: third,
+                password: 'pw-test-1234',
+            });
+            ownerId = (await env.server.stores.user.getByUsername(
+                env.users.user.username,
+            ))!.id!;
+        });
+
+        const shareAt = (prefix: string, body: Record<string, unknown>) =>
+            req('POST', '/kv/shares', env.users.user.token, {
+                prefix,
+                ...body,
+            });
+
+        /** Leaves `left` units in one of the owner's windows. */
+        const drain = (
+            window: { scope: string; limit: number; window: number },
+            left = 0,
+        ) =>
+            claimRateLimit(
+                `${window.scope}:${ownerId}`,
+                window.limit,
+                window.window,
+                window.limit - left,
+            );
+
+        it('spends neither the mint nor the grant budget when every share stays as it is', async () => {
+            const prefix = `again-${uuidv4().slice(0, 8)}:`;
+            const recipients = [env.users.other.username, third];
+            const made = (await (
+                await shareAt(prefix, { recipients, label: 'Room' })
+            ).json()) as { items: Array<{ handle: string }> };
+            await drain(EVENTS_KV_HANDLE_LIMIT);
+            for (const window of KV_SHARE_GRANT_LIMITS) await drain(window);
+            try {
+                const again = await shareAt(prefix, {
+                    recipients: [...recipients, third],
+                });
+                expect(again.status).toBe(200);
+                const { items } = (await again.json()) as {
+                    items: Array<{ handle: string; label: string }>;
+                };
+                expect(items.map((item) => item.handle)).toEqual([
+                    made.items[0].handle,
+                    made.items[1].handle,
+                    made.items[1].handle,
+                ]);
+                expect(items[0].label).toBe('Room');
+
+                const single = await shareAt(prefix, {
+                    recipient: env.users.other.username,
+                    access: 'read',
+                });
+                expect(single.status).toBe(200);
+            } finally {
+                await resetBudgets(env);
+            }
+        });
+
+        it('spends the mint budget on a change, and grant units only on recipients without the share', async () => {
+            const prefix = `again-${uuidv4().slice(0, 8)}:`;
+            const other = env.users.other.username;
+            expect((await shareAt(prefix, { recipient: other })).status).toBe(
+                200,
+            );
+            for (const window of KV_SHARE_GRANT_LIMITS) await drain(window);
+            try {
+                // A level change on a standing share takes no grant unit.
+                const raised = await shareAt(prefix, {
+                    recipients: [other],
+                    access: 'write',
+                });
+                expect(raised.status).toBe(200);
+
+                // Naming an unknown user next to it costs what a new share
+                // would, so the window refuses both.
+                for (const extra of ['nobody-by-this-name', third]) {
+                    const refused = await shareAt(prefix, {
+                        recipients: [other, extra],
+                        access: 'write',
+                    });
+                    expect(refused.status).toBe(429);
+                }
+            } finally {
+                await resetBudgets(env);
+            }
+
+            await drain(EVENTS_KV_HANDLE_LIMIT);
+            try {
+                const lowered = await shareAt(prefix, {
+                    recipient: other,
+                    access: 'read',
+                });
+                expect(lowered.status).toBe(429);
+                await expect(lowered.json()).resolves.toMatchObject({
+                    code: 'too_many_requests',
+                });
+            } finally {
+                await resetBudgets(env);
+            }
+        });
+
+        it('counts turning notify on or off as a change', async () => {
+            const prefix = `again-${uuidv4().slice(0, 8)}:`;
+            const other = env.users.other.username;
+            expect((await shareAt(prefix, { recipient: other })).status).toBe(
+                200,
+            );
+            await drain(EVENTS_KV_HANDLE_LIMIT);
+            try {
+                expect(
+                    (await shareAt(prefix, { recipient: other })).status,
+                ).toBe(200);
+                expect(
+                    (
+                        await shareAt(prefix, {
+                            recipient: other,
+                            notify: true,
+                        })
+                    ).status,
+                ).toBe(429);
+            } finally {
+                await resetBudgets(env);
+            }
+        });
+
+        it('spends the lookup budget on every recipient named', async () => {
+            const prefix = `again-${uuidv4().slice(0, 8)}:`;
+            const other = env.users.other.username;
+            expect((await shareAt(prefix, { recipient: other })).status).toBe(
+                200,
+            );
+            await resetBudgets(env);
+            await drain(KV_SHARE_LOOKUP_LIMIT, 1);
+            try {
+                expect(
+                    (await shareAt(prefix, { recipient: other })).status,
+                ).toBe(200);
+                const refused = await shareAt(prefix, { recipient: other });
+                expect(refused.status).toBe(429);
+                await expect(refused.json()).resolves.toMatchObject({
+                    code: 'too_many_requests',
+                });
+            } finally {
+                await resetBudgets(env);
+            }
+        });
+    });
 });
 
 describe('/events/kv-handles beside /kv/shares', () => {

src/backend/controllers/kvShare/KvShareController.ts

@@ -69,6 +69,7 @@ const shareBody = (share: KvShareView) => ({
     appUid: publicAppUid(share.appUid),
     holder: share.holderUsername,
     label: share.label,
+    notify: share.notify,
     createdAt: share.createdAt,
 });
 
@@ -128,8 +129,8 @@ export class KvShareController extends PuterController {
     /**
      * POST /kv/shares — share a region with another account, `read` unless the
      * body asks otherwise. Sharing the same region with the same account again
-     * changes its level or label in place. `recipients` shares it with several
-     * at once and reports each one on its own item.
+     * changes its level, label or `notify` flag in place. `recipients` shares
+     * it with several at once and reports each one on its own item.
      */
     @Post('/shares', {
         subdomain: 'api',
@@ -145,6 +146,7 @@ export class KvShareController extends PuterController {
             access: body.access,
             appUid: body.appUid,
             label: body.label,
+            notify: body.notify,
         };
         const recipients = batchRecipients(body);
         if (recipients) {
@@ -206,6 +208,7 @@ export class KvShareController extends PuterController {
                 appUid: publicAppUid(row.appUid),
                 holder: row.holderUsername,
                 label: row.label,
+                notify: row.notify,
                 createdAt: row.createdAt,
                 revokedAt: row.revokedAt,
                 revokedBy: row.revokedBy,
@@ -234,6 +237,7 @@ export class KvShareController extends PuterController {
                 access: row.access,
                 appUid: publicAppUid(row.appUid),
                 label: row.label,
+                notify: row.notify,
                 createdAt: row.createdAt,
             })),
             ...(page.cursor ? { cursor: page.cursor } : {}),

src/backend/controllers/notification/NotificationController.http.test.ts

@@ -21,9 +21,9 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
 
 /**
- * Route-level coverage for the notification endpoint. Dismissing is the only
- * mark the desktop performs over HTTP, so this suite is what catches the route
- * going missing or a gate turning a legitimate dismiss away.
+ * Route-level coverage for the notification endpoints the desktop calls over
+ * HTTP, so this suite is what catches a route going missing or a gate turning
+ * a legitimate dismiss or removal away.
  */
 describe('notification endpoints over HTTP', () => {
     let env: PuterTestEnv;
@@ -85,4 +85,34 @@ describe('notification endpoints over HTTP', () => {
         });
         expect(after?.acknowledged).toBeFalsy();
     });
+
+    it('removes a notification', async () => {
+        const owner = env.users.user;
+        const { userId, uid } = await seedNotification(owner.username);
+
+        const res = await post('/notif/delete', owner.token, { uid });
+
+        expect(res.status).toBe(200);
+        await expect(res.json()).resolves.toEqual({});
+        const after = await env.server.stores.notification.getByUid(uid, {
+            userId,
+        });
+        expect(after).toBeNull();
+    });
+
+    it('refuses an anonymous removal', async () => {
+        const { userId, uid } = await seedNotification(env.users.other.username);
+
+        const res = await fetch(new URL('/notif/delete', env.apiOrigin), {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ uid }),
+        });
+
+        expect(res.status).toBeGreaterThanOrEqual(400);
+        const after = await env.server.stores.notification.getByUid(uid, {
+            userId,
+        });
+        expect(after?.uid).toBe(uid);
+    });
 });

src/backend/controllers/notification/NotificationController.test.ts

@@ -183,3 +183,74 @@ describe('NotificationController.markAck', () => {
         expect(after?.acknowledged).toBeFalsy();
     });
 });
+
+// ── /notif/delete ───────────────────────────────────────────────────
+
+describe('NotificationController.remove', () => {
+    it('deletes the underlying notification row', async () => {
+        const { actor, userId } = await makeUser();
+        const created = await server.stores.notification.create({
+            userId,
+            value: { title: 't' },
+        });
+
+        const { res, captured } = makeRes();
+        await controller.remove(
+            makeReq({ body: { uid: created.uid }, actor }),
+            res,
+        );
+
+        expect(captured.body).toEqual({});
+        const after = await server.stores.notification.getByUid(
+            created.uid as string,
+            { userId },
+        );
+        expect(after).toBeNull();
+    });
+
+    it('succeeds for a uid that names no row', async () => {
+        const { actor } = await makeUser();
+        const { res, captured } = makeRes();
+        await controller.remove(
+            makeReq({ body: { uid: 'no-such-uid' }, actor }),
+            res,
+        );
+        expect(captured.body).toEqual({});
+    });
+
+    it('rejects a missing uid with 400', async () => {
+        const { actor } = await makeUser();
+        const { res } = makeRes();
+        await expect(
+            controller.remove(makeReq({ body: {}, actor }), res),
+        ).rejects.toMatchObject({ statusCode: 400 });
+    });
+
+    it('throws 401 when there is no actor on the request', async () => {
+        const { res } = makeRes();
+        await expect(
+            controller.remove(makeReq({ body: { uid: 'whatever' } }), res),
+        ).rejects.toMatchObject({ statusCode: 401 });
+    });
+
+    it("does not delete another user's notification", async () => {
+        const a = await makeUser();
+        const b = await makeUser();
+        const created = await server.stores.notification.create({
+            userId: a.userId,
+            value: {},
+        });
+
+        const { res } = makeRes();
+        await controller.remove(
+            makeReq({ body: { uid: created.uid }, actor: b.actor }),
+            res,
+        );
+
+        const after = await server.stores.notification.getByUid(
+            created.uid as string,
+            { userId: a.userId },
+        );
+        expect(after?.uid).toBe(created.uid);
+    });
+});

src/backend/controllers/notification/NotificationController.ts

@@ -20,16 +20,44 @@
 import type { Request, Response } from 'express';
 import { Controller, Post } from '../../core/http/decorators.js';
 import { HttpError } from '../../core/http/HttpError.js';
-import type { NotificationService } from '../../services/notification/NotificationService.js';
 import { PuterController } from '../types.js';
 
+/** Both routes fire per interaction; the ceiling catches a loop, not a user. */
+const MARK_ROUTE_OPTIONS = {
+    subdomain: 'api',
+    requireUserActor: true,
+    allowFullAccessToken: true,
+    rateLimit: {
+        scope: 'notification-mark',
+        limit: 300,
+        window: 60_000,
+        key: 'user',
+    },
+} as const;
+
+/** The uid and owner a mark names, or the 400/401 the request earns. */
+const markTarget = (req: Request): { uid: string; userId: number } => {
+    const uid = req.body?.uid;
+    if (typeof uid !== 'string' || uid.length === 0) {
+        throw new HttpError(400, '`uid` must be a non-empty string', {
+            legacyCode: 'bad_request',
+        });
+    }
+    const userId = req.actor?.user?.id;
+    if (!userId)
+        throw new HttpError(401, 'Unauthorized', {
+            legacyCode: 'unauthorized',
+        });
+    return { uid, userId };
+};
+
 /**
- * GUI-facing notification endpoint. Supplements the `puter-notifications`
- * driver (which handles CRUD via `/drivers/call`) with the one mutation route
- * the puter desktop client calls directly.
+ * GUI-facing notification endpoints. Supplement the `puter-notifications`
+ * driver (which handles reads via `/drivers/call`) with the mutations the puter
+ * desktop client calls directly.
  *
- * It emits `outer.gui.notif.ack` via the NotificationService so other open tabs
- * for the same user see the state change immediately.
+ * Each goes through the NotificationService, which tells the user's other open
+ * tabs (`notif.ack`, `notif.removed`) so they update immediately.
  */
 @Controller('/notif')
 export class NotificationController extends PuterController {
@@ -37,50 +65,21 @@ export class NotificationController extends PuterController {
      * POST /notif/mark-ack — user dismissed a notification. Sets `acknowledged`
      * timestamp; pushes ack event to sockets.
      */
-    @Post('/mark-ack', {
-        subdomain: 'api',
-        requireUserActor: true,
-        allowFullAccessToken: true,
-        // Fires per notification interaction, so the ceiling stays
-        // generous — it is here to catch a loop, not to pace a user.
-        rateLimit: {
-            scope: 'notification-mark',
-            limit: 300,
-            window: 60_000,
-            key: 'user',
-        },
-    })
+    @Post('/mark-ack', MARK_ROUTE_OPTIONS)
     async markAck(req: Request, res: Response): Promise<void> {
-        const uid = req.body?.uid;
-        if (typeof uid !== 'string' || uid.length === 0) {
-            throw new HttpError(400, '`uid` must be a non-empty string', {
-                legacyCode: 'bad_request',
-            });
-        }
-        const userId = req.actor?.user?.id;
-        if (!userId)
-            throw new HttpError(401, 'Unauthorized', {
-                legacyCode: 'unauthorized',
-            });
+        const { uid, userId } = markTarget(req);
+        await this.services.notification.markAcknowledged(uid, userId);
+        res.json({});
+    }
 
-        const notifService = this.services.notification as unknown as
-            | NotificationService
-            | undefined;
-        if (notifService?.markAcknowledged) {
-            await notifService.markAcknowledged(uid, userId);
-        } else {
-            // Fallback: direct store call if service isn't wired
-            await (
-                this.stores as Record<string, unknown> as {
-                    notification: {
-                        markAcknowledged: (
-                            uid: string,
-                            userId: number,
-                        ) => Promise<boolean>;
-                    };
-                }
-            ).notification.markAcknowledged(uid, userId);
-        }
+    /**
+     * POST /notif/delete — user removed a notification from their history. A
+     * uid naming none of their rows succeeds too: it is gone either way.
+     */
+    @Post('/delete', MARK_ROUTE_OPTIONS)
+    async remove(req: Request, res: Response): Promise<void> {
+        const { uid, userId } = markTarget(req);
+        await this.services.notification.remove(uid, userId);
         res.json({});
     }
 }

src/backend/controllers/puterai/PuterAIController.test.ts

@@ -511,6 +511,31 @@ describe('PuterAIController.openaiChatCompletions', () => {
 
         const completeArgs = completeSpy.mock.calls[0]![0];
         expect(completeArgs.provider).toBe('openai-completion');
+        expect(completeArgs.fallback).toBeUndefined();
+    });
+
+    it('forwards fallback: false so the driver stops at the first route', async () => {
+        const completeSpy = stubChatComplete({
+            message: { role: 'assistant', content: 'hi there' },
+            finish_reason: 'stop',
+            usage: { prompt_tokens: 4, completion_tokens: 2 },
+        });
+
+        const { res } = makeRes();
+        await controller.openaiChatCompletions(
+            makeReq({
+                body: {
+                    model: 'gpt-test',
+                    provider: 'openai-completion',
+                    fallback: false,
+                    messages: [{ role: 'user', content: 'hi' }],
+                },
+                actor: makeUserActor(),
+            }),
+            res,
+        );
+
+        expect(completeSpy.mock.calls[0]![0].fallback).toBe(false);
     });
 
     it('still pins OpenAI when the caller names no model, so the default model does not move', async () => {

src/backend/controllers/puterai/PuterAIController.ts

@@ -448,6 +448,7 @@ export class PuterAIController extends PuterController {
                   }
                 : {}),
             ...openaiCompatProvider(body),
+            ...(body.fallback === false ? { fallback: false } : {}),
         };
 
         const result = await this.#complete(res, completeArgs);
@@ -545,8 +546,7 @@ export class PuterAIController extends PuterController {
                         }
                     } else if (ev.type === 'usage') {
                         usageDetails = ev.usageDetails as
-                            | UsageDetails
-                            | undefined;
+                            UsageDetails | undefined;
                         finishReason = ev.finish_reason as string | undefined;
                     } else if (ev.type === 'error') {
                         res.write(
@@ -649,6 +649,7 @@ export class PuterAIController extends PuterController {
                 : {}),
             ...(finiteMaxTokens(body.max_tokens) ?? {}),
             ...openaiCompatProvider(body),
+            ...(body.fallback === false ? { fallback: false } : {}),
         };
 
         const completionId = `cmpl-${randomId()}`;
@@ -694,8 +695,7 @@ export class PuterAIController extends PuterController {
                         sendChunk(ev.text);
                     } else if (ev.type === 'usage') {
                         usageDetails = ev.usageDetails as
-                            | UsageDetails
-                            | undefined;
+                            UsageDetails | undefined;
                         finishReason = ev.finish_reason as string | undefined;
                     } else if (ev.type === 'error') {
                         res.write(
@@ -749,8 +749,7 @@ export class PuterAIController extends PuterController {
                     text: extractTextContent(
                         (
                             messageResult.message as
-                                | Record<string, unknown>
-                                | undefined
+                                Record<string, unknown> | undefined
                         )?.content,
                     ),
                     index: 0,
@@ -869,6 +868,7 @@ export class PuterAIController extends PuterController {
                   }
                 : {}),
             provider: providerName,
+            ...(body.fallback === false ? { fallback: false } : {}),
         };
 
         const responseId = generateId('resp');

src/backend/controllers/puterai/anthropicWire.test.ts

@@ -324,6 +324,18 @@ describe('parseAnthropicRequest', () => {
         expect(args.provider).toBe('claude');
         expect(args.normalize).toBe(false);
         expect(args.streamToolInput).toBe(true);
+        expect(args.fallback).toBeUndefined();
+    });
+
+    it('forwards fallback: false and nothing else for fallback', () => {
+        expect(
+            parseAnthropicRequest({ messages: [], fallback: false }, headers)
+                .fallback,
+        ).toBe(false);
+        expect(
+            parseAnthropicRequest({ messages: [], fallback: 'no' }, headers)
+                .fallback,
+        ).toBeUndefined();
     });
 
     it('maps thinking, tool_choice, stop_sequences, top_p/top_k', () => {

src/backend/controllers/puterai/anthropicWire.ts

@@ -431,6 +431,7 @@ export const parseAnthropicRequest = (
             : {}),
         ...(betas.length ? { anthropicBetas: betas } : {}),
         provider: toStringOrEmpty(body.provider) || 'claude',
+        ...(body.fallback === false ? { fallback: false } : {}),
     };
     // Silently ignored: metadata, service_tier, inference_geo, container,
     // fallbacks, diagnostics.

src/backend/core/context.ts

@@ -87,6 +87,11 @@ export interface KnownContextFields {
      * different provider's translation of it.
      */
     strictUpstreamErrors: boolean;
+    /**
+     * Set while Puter files something into an account for it (mail delivery),
+     * so filesystem events never report that write as the request actor's own.
+     */
+    platformWrite: boolean;
 }
 
 // Every key of `KnownContextFields`, checked both ways by the compiler, so
@@ -100,6 +105,7 @@ const KNOWN_KEY_FLAGS: Record<keyof KnownContextFields, true> = {
     keepDriverSlot: true,
     upstreamSignal: true,
     strictUpstreamErrors: true,
+    platformWrite: true,
 };
 const KNOWN_KEYS: ReadonlySet<string> = new Set(Object.keys(KNOWN_KEY_FLAGS));
 

src/backend/core/http/middleware/gates.test.ts

@@ -836,6 +836,34 @@ describe('requireVerifiedAccount', () => {
         expectHttpError(got, 403, 'card_verification_required');
     });
 
+    it('asks an email-gate-last account for card, then phone, then email', () => {
+        const pending = {
+            uuid: 'u-1',
+            requires_email_confirmation: true,
+            email_confirmed: false,
+            requires_phone_verification: true,
+            requires_card_verification: true,
+            metadata: { emailGateLast: true },
+        };
+        const gateFor = (user: Record<string, unknown>) =>
+            runGate(requireVerifiedAccount(), { actor: { user } });
+        expectHttpError(gateFor(pending), 403, 'card_verification_required');
+        expectHttpError(
+            gateFor({ ...pending, requires_card_verification: false }),
+            403,
+            'phone_verification_required',
+        );
+        expectHttpError(
+            gateFor({
+                ...pending,
+                requires_card_verification: false,
+                requires_phone_verification: false,
+            }),
+            403,
+            'email_confirmation_required',
+        );
+    });
+
     it('returns 403 two_factor_required for a seat whose team requires 2FA', async () => {
         const teams = { getOrgSeat: async () => ({ require_2fa: 1 }) };
         const got = await runGateAsync(requireTeam2fa(teams), {

src/backend/core/http/middleware/gates.ts

@@ -32,6 +32,7 @@ import {
     type CardFallbackDeps,
 } from '../../../util/cardFallback';
 import { reserveUsernames } from '../../../util/reservedUsernames';
+import { isEmailGateLast } from '../../../util/verificationOrder';
 import {
     isAccountContext,
     isAppActor,
@@ -309,11 +310,25 @@ export const requireVerifiedAccount = (): RequestHandler =>
 export const assertVerifiedAccount = (
     user: AccountGateUser | undefined,
 ): void => {
-    if (user?.requires_email_confirmation && !user?.email_confirmed) {
-        throw new HttpError(403, 'Please confirm your email to continue', {
-            legacyCode: 'email_confirmation_required',
-        });
-    }
+    const assertEmail = () => {
+        if (user?.requires_email_confirmation && !user?.email_confirmed) {
+            throw new HttpError(403, 'Please confirm your email to continue', {
+                legacyCode: 'email_confirmation_required',
+            });
+        }
+    };
+    const assertCard = () => {
+        if (user?.requires_card_verification) {
+            throw new HttpError(403, 'Please verify your card to continue', {
+                legacyCode: 'card_verification_required',
+            });
+        }
+    };
+    // Email → phone → card, or card → phone → email for an email-gate-last
+    // account, so the client is sent to the gate it has to clear next.
+    const emailLast = isEmailGateLast(user);
+    if (emailLast) assertCard();
+    else assertEmail();
     if (user?.requires_phone_verification) {
         throw new HttpError(
             403,
@@ -323,11 +338,8 @@ export const assertVerifiedAccount = (
             },
         );
     }
-    if (user?.requires_card_verification) {
-        throw new HttpError(403, 'Please verify your card to continue', {
-            legacyCode: 'card_verification_required',
-        });
-    }
+    if (emailLast) assertEmail();
+    else assertCard();
     // A team seat signs in on a password its administrator still holds,
     // so it reaches nothing until it has replaced that password.
     if (user?.requires_password_change) {

src/backend/core/http/middleware/largeJsonBody.test.ts

@@ -0,0 +1,163 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { EventEmitter } from 'node:events';
+import type { NextFunction, Request, RequestHandler, Response } from 'express';
+import { describe, expect, it, vi } from 'vitest';
+import { isHttpError } from '../HttpError';
+import {
+    createLargeJsonBodyParser,
+    DEFAULT_JSON_LIMIT_BYTES,
+} from './largeJsonBody';
+
+const MiB = 1024 * 1024;
+
+// A complete request with no socket reads as already consumed to body-parser,
+// so the parsers call straight through, leaving any preset `body` in place.
+const makeReq = (
+    bytes: number | undefined,
+    user: string | null = 'u1',
+    body?: unknown,
+) =>
+    ({
+        complete: true,
+        body,
+        headers: {
+            'content-type': 'application/json',
+            ...(bytes === undefined ? {} : { 'content-length': String(bytes) }),
+            ...(user ? { authorization: user } : {}),
+        },
+    }) as unknown as Request;
+
+const makeRes = () => {
+    const res = new EventEmitter() as EventEmitter & {
+        setHeader: ReturnType<typeof vi.fn>;
+    };
+    res.setHeader = vi.fn();
+    return res;
+};
+
+const authProbe = vi.fn<RequestHandler>((req, _res, next) => {
+    if (req.headers.authorization) {
+        (req as { actor?: unknown }).actor = {
+            user: { uuid: req.headers.authorization },
+        };
+    }
+    next();
+});
+
+const run = (mw: RequestHandler, req: Request, res = makeRes()) =>
+    new Promise<{ err: unknown; res: ReturnType<typeof makeRes> }>(
+        (resolve) => {
+            mw(
+                req,
+                res as unknown as Response,
+                ((err?: unknown) => resolve({ err, res })) as NextFunction,
+            );
+        },
+    );
+
+const statusOf = (err: unknown) => (isHttpError(err) ? err.statusCode : err);
+
+describe('createLargeJsonBodyParser', () => {
+    const create = ({
+        budgetBytes = 200 * MiB,
+        perUserBytes = 128 * MiB,
+        accepts,
+    }: {
+        budgetBytes?: number;
+        perUserBytes?: number;
+        accepts?: (req: Request) => boolean;
+    } = {}) =>
+        createLargeJsonBodyParser({
+            authProbe,
+            limitBytes: 128 * MiB,
+            budgetBytes,
+            perUserBytes,
+            accepts,
+        });
+
+    it.each([
+        ['no Content-Length', undefined],
+        ['a body within the default limit', DEFAULT_JSON_LIMIT_BYTES],
+        ['a body over the large limit', 128 * MiB + 1],
+    ])('leaves %s to the global parser without probing', async (_, bytes) => {
+        authProbe.mockClear();
+        const { err } = await run(create(), makeReq(bytes));
+        expect(err).toBeUndefined();
+        expect(authProbe).not.toHaveBeenCalled();
+    });
+
+    it('leaves an unauthenticated large body to the global parser', async () => {
+        const mw = create({ budgetBytes: 60 * MiB });
+        const { err } = await run(mw, makeReq(60 * MiB, null));
+        expect(err).toBeUndefined();
+        // Nothing was charged, so a signed-in caller still fits.
+        expect((await run(mw, makeReq(60 * MiB))).err).toBeUndefined();
+    });
+
+    it('charges nothing for a client that left during the probe', async () => {
+        const mw = create({ budgetBytes: 60 * MiB });
+        const gone = Object.assign(makeRes(), { closed: true });
+        expect((await run(mw, makeReq(60 * MiB), gone)).err).toBeUndefined();
+        expect((await run(mw, makeReq(60 * MiB))).err).toBeUndefined();
+    });
+
+    it('refuses with 503 past the budget until a response closes', async () => {
+        const mw = create({ budgetBytes: 130 * MiB });
+        const first = await run(mw, makeReq(60 * MiB, 'u1'));
+        await run(mw, makeReq(60 * MiB, 'u2'));
+
+        const refused = await run(mw, makeReq(60 * MiB, 'u3'));
+        expect(statusOf(refused.err)).toBe(503);
+        // Load shedding is recorded, not paged.
+        expect(isHttpError(refused.err) && refused.err.alarmSeverity).toBe(
+            'info',
+        );
+        expect(refused.res.setHeader).toHaveBeenCalledWith('Retry-After', '5');
+
+        first.res.emit('close');
+        expect((await run(mw, makeReq(60 * MiB, 'u3'))).err).toBeUndefined();
+    });
+
+    it("refuses one user's bodies with 429 past their share", async () => {
+        const mw = create({ perUserBytes: 100 * MiB });
+        const first = await run(mw, makeReq(60 * MiB, 'u1'));
+
+        const refused = await run(mw, makeReq(60 * MiB, 'u1'));
+        expect(statusOf(refused.err)).toBe(429);
+        expect(refused.res.setHeader).toHaveBeenCalledWith('Retry-After', '5');
+        // Another user isn't held back by it.
+        expect((await run(mw, makeReq(60 * MiB, 'u2'))).err).toBeUndefined();
+
+        first.res.emit('close');
+        expect((await run(mw, makeReq(60 * MiB, 'u1'))).err).toBeUndefined();
+    });
+
+    it('refuses with 413 a parsed body `accepts` turns down', async () => {
+        const mw = create({
+            accepts: (req) => (req.body as { ok?: boolean }).ok === true,
+        });
+        const refused = await run(mw, makeReq(60 * MiB, 'u1', { ok: false }));
+        expect(statusOf(refused.err)).toBe(413);
+        expect(
+            (await run(mw, makeReq(60 * MiB, 'u2', { ok: true }))).err,
+        ).toBeUndefined();
+    });
+});

src/backend/core/http/middleware/largeJsonBody.ts

@@ -0,0 +1,134 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import express, { type Request, type RequestHandler } from 'express';
+import { actorUid } from '../../actor';
+import { HttpError } from '../HttpError';
+
+/** The global JSON parser's limit, which applies before auth. */
+export const DEFAULT_JSON_LIMIT_BYTES = 50 * 1024 * 1024;
+
+interface LargeJsonBodyOptions {
+    /** The global auth probe; it skips requests that already carry an actor. */
+    authProbe: RequestHandler;
+    limitBytes: number;
+    /** Declared bytes of large bodies a process holds at once. */
+    budgetBytes: number;
+    /** Declared bytes of large bodies one user holds at once. */
+    perUserBytes: number;
+    /** Checked once the body is parsed; a body it refuses gets a 413. */
+    accepts?: (req: Request) => boolean;
+}
+
+/**
+ * Parses JSON bodies over the default limit, up to `limitBytes`, once the
+ * caller's header or cookie credential resolves. Unauthenticated requests and
+ * requests without a Content-Length fall through to the global parser and its
+ * limit.
+ *
+ * The parsed body lives as long as the request, so a body's bytes count against
+ * both budgets until its response closes. Over the user's budget gets a 429,
+ * over the process budget a 503.
+ */
+export const createLargeJsonBodyParser = ({
+    authProbe,
+    limitBytes,
+    budgetBytes,
+    perUserBytes,
+    accepts,
+}: LargeJsonBodyOptions): RequestHandler => {
+    const json = express.json({ limit: limitBytes });
+    const textAsJson = express.json({
+        limit: limitBytes,
+        type: (req) =>
+            req.headers['content-type'] === 'text/plain;actually=json',
+    });
+    let inFlightBytes = 0;
+    const inFlightByUser = new Map<string, number>();
+
+    return (req, res, next) => {
+        const declared = Number(req.headers['content-length']);
+        if (!(declared > DEFAULT_JSON_LIMIT_BYTES && declared <= limitBytes)) {
+            next();
+            return;
+        }
+        void authProbe(req, res, () => {
+            // `res.closed`: the client left during the probe, so `close` has
+            // already fired and would never release a charge.
+            if (!req.actor || res.closed) {
+                next();
+                return;
+            }
+            const user = req.actor.user.uuid ?? actorUid(req.actor);
+            const userBytes = inFlightByUser.get(user) ?? 0;
+            if (userBytes + declared > perUserBytes) {
+                res.setHeader('Retry-After', '5');
+                next(
+                    new HttpError(
+                        429,
+                        'Too many large requests in progress for this account',
+                        { legacyCode: 'too_many_requests' },
+                    ),
+                );
+                return;
+            }
+            if (inFlightBytes + declared > budgetBytes) {
+                res.setHeader('Retry-After', '5');
+                next(
+                    new HttpError(503, 'Too many large requests in progress', {
+                        code: 'large_body_capacity',
+                        // Load shedding, not a fault: record it, don't page.
+                        alarmSeverity: 'info',
+                    }),
+                );
+                return;
+            }
+            inFlightBytes += declared;
+            inFlightByUser.set(user, userBytes + declared);
+            res.once('close', () => {
+                inFlightBytes -= declared;
+                const left = (inFlightByUser.get(user) ?? 0) - declared;
+                if (left > 0) inFlightByUser.set(user, left);
+                else inFlightByUser.delete(user);
+            });
+            json(req, res, (err?: unknown) => {
+                if (err) {
+                    next(err);
+                    return;
+                }
+                textAsJson(req, res, (err?: unknown) => {
+                    if (err) {
+                        next(err);
+                        return;
+                    }
+                    if (accepts && !accepts(req)) {
+                        // The same answer the default limit gives.
+                        next(
+                            new HttpError(413, 'request entity too large', {
+                                legacyCode: 'client_bad_request',
+                            }),
+                        );
+                        return;
+                    }
+                    next();
+                });
+            });
+        });
+    };
+};

src/backend/core/http/middleware/rateLimit.js

@@ -84,6 +84,10 @@ export function sweepMemoryWindows() {
         )
             memoryWindows.delete(k);
     }
+    for (const [k, entry] of memoryCounters) {
+        if (entry.index < Math.floor(now / entry.windowMs) - 1)
+            memoryCounters.delete(k);
+    }
 }
 
 {
@@ -155,6 +159,97 @@ async function peekMemory(key, limit, windowMs) {
     return timestamps.length < limit;
 }
 
+// -- Counted claims --------------------------------------------------
+//
+// `claim` keeps a member per unit, which is fine for a few units and not for
+// thousands. The counted form keeps two counters per key, the current fixed
+// window and the one before it, and slides by weighting the previous one by
+// how much of it the window still covers. It can be off by a little at window
+// edges, and its buckets are separate from the other forms'.
+
+/** The sliding-window estimate from the two fixed windows it overlaps. */
+const slidingCount = (previous, current, now, windowMs) =>
+    previous * (1 - (now % windowMs) / windowMs) + current;
+
+/** How many of `units` a counted claim that left `used` spent may keep. */
+const countedAdmits = (used, limit, units) =>
+    units - Math.min(units, Math.max(0, Math.ceil(used - limit)));
+
+/** Key → `{ index, current, previous, windowMs }`, swept with the windows. */
+const memoryCounters = new Map();
+
+async function claimCountMemory(key, limit, windowMs, units) {
+    const now = Date.now();
+    const index = Math.floor(now / windowMs);
+    let entry = memoryCounters.get(key);
+    if (!entry) {
+        if (memoryCounters.size >= MEMORY_MAX_KEYS) {
+            memoryCounters.delete(memoryCounters.keys().next().value);
+        }
+        entry = { index, current: 0, previous: 0, windowMs };
+        memoryCounters.set(key, entry);
+    } else if (entry.index !== index) {
+        entry.previous = entry.index === index - 1 ? entry.current : 0;
+        entry.current = 0;
+        entry.index = index;
+    }
+    entry.windowMs = windowMs;
+    const used = slidingCount(entry.previous, entry.current, now, windowMs);
+    const admitted = countedAdmits(used + units, limit, units);
+    entry.current += admitted;
+    return admitted;
+}
+
+async function claimCountRedis(redis, key, limit, windowMs, units) {
+    const now = Date.now();
+    const index = Math.floor(now / windowMs);
+    // The hash tag keeps both windows in one slot, so one MULTI reads both.
+    const current = `ratec:{${key}}:${index}`;
+    const results = await redis
+        .multi()
+        .incrby(current, units)
+        .pexpire(current, 2 * windowMs)
+        .get(`ratec:{${key}}:${index - 1}`)
+        .exec();
+    const used = slidingCount(
+        Number(multiResult(results, 2) ?? 0),
+        Number(multiResult(results, 0)),
+        now,
+        windowMs,
+    );
+    const admitted = countedAdmits(used, limit, units);
+    if (admitted < units) await redis.decrby(current, units - admitted);
+    return admitted;
+}
+
+async function claimCountKv(kv, key, limit, windowMs, units) {
+    const now = Date.now();
+    const index = Math.floor(now / windowMs);
+    const current = `ratec:${key}:${index}`;
+    const [counted, previous] = await Promise.all([
+        kv.incr({
+            key: current,
+            pathAndAmountMap: { '': units },
+            expireAt: Math.ceil((now + 2 * windowMs) / 1000),
+        }),
+        kv.get({ key: `ratec:${key}:${index - 1}` }),
+    ]);
+    const used = slidingCount(
+        Number(previous.res ?? 0),
+        Number(counted.res),
+        now,
+        windowMs,
+    );
+    const admitted = countedAdmits(used, limit, units);
+    if (admitted < units) {
+        await kv.decr({
+            key: current,
+            pathAndAmountMap: { '': units - admitted },
+        });
+    }
+    return admitted;
+}
+
 // -- Redis backend ---------------------------------------------------
 
 // ioredis MULTI/EXEC reports per-command failures inside the exec() result
@@ -475,6 +570,10 @@ function instrumentBackendPair(name, pair) {
             withSpan('rate_limit.claim', attrs, () =>
                 pair.claim(key, limit, windowMs, units),
             ),
+        count: (key, limit, windowMs, units) =>
+            withSpan('rate_limit.count', attrs, () =>
+                pair.count(key, limit, windowMs, units),
+            ),
         acquire: (key, limit) =>
             withSpan('rate_limit.acquire', attrs, () =>
                 pair.acquire(key, limit),
@@ -486,6 +585,7 @@ const memoryBackendPair = instrumentBackendPair('memory', {
     rate: checkMemory,
     peek: peekMemory,
     claim: claimMemory,
+    count: claimCountMemory,
     acquire: acquireMemoryConcurrent,
 });
 
@@ -536,6 +636,8 @@ export function configureRateLimit({
                 peekRedis(redis, key, limit, windowMs),
             claim: (key, limit, windowMs, units) =>
                 claimRedis(redis, key, limit, windowMs, units),
+            count: (key, limit, windowMs, units) =>
+                claimCountRedis(redis, key, limit, windowMs, units),
             acquire: (key, limit) => acquireRedisConcurrent(redis, key, limit),
         });
     }
@@ -545,6 +647,8 @@ export function configureRateLimit({
             peek: (key, limit) => peekKv(kv, key, limit),
             claim: (key, limit, windowMs, units) =>
                 claimKv(kv, key, limit, windowMs, units),
+            count: (key, limit, windowMs, units) =>
+                claimCountKv(kv, key, limit, windowMs, units),
             acquire: (key, limit) => acquireKvConcurrent(kv, key, limit),
         });
     }
@@ -874,6 +978,32 @@ export async function claimRateLimit(key, limit, windowMs, units, backend) {
     }
 }
 
+/**
+ * `claimRateLimit` for budgets measured in many units per call: a claim costs
+ * the same however many units it spends (see "Counted claims" above). Its
+ * buckets are its own, so a key is only ever claimed one way. Fails open,
+ * admitting every unit.
+ */
+export async function claimCountedRateLimit(
+    key,
+    limit,
+    windowMs,
+    units,
+    backend,
+) {
+    if (!(units > 0)) return 0;
+    const bk = resolveBackend(backend);
+    try {
+        return await bk.count(key, limit, windowMs, units);
+    } catch (err) {
+        console.error(
+            '[rate-limit] imperative counted claim failed, failing open:',
+            err,
+        );
+        return units;
+    }
+}
+
 /**
  * Charge a route-shaped rate-limit spec from inside a handler.
  *

src/backend/core/http/middleware/rateLimit.test.js

@@ -37,6 +37,7 @@ import {
     acquireDriverConcurrent,
     checkDriverRateLimit,
     checkRateLimit,
+    claimCountedRateLimit,
     claimRateLimit,
     concurrencyGate,
     consumeRouteRateLimit,
@@ -1487,6 +1488,13 @@ describe('rate + concurrent limiting — kv backend', () => {
         expect(await claimRateLimit(key, 5, 60_000, 1)).toBe(0);
     });
 
+    it('claims counted units, admitting only what fits', async () => {
+        const key = uniqueScope();
+        expect(await claimCountedRateLimit(key, 5, 60_000, 3)).toBe(3);
+        expect(await claimCountedRateLimit(key, 5, 60_000, 3)).toBe(2);
+        expect(await claimCountedRateLimit(key, 5, 60_000, 1)).toBe(0);
+    });
+
     it('claims against a full bucket as full while expired rows are unswept', async () => {
         const key = uniqueScope();
         await seedUnswept(`rate:${key}:`, 3);
@@ -2010,6 +2018,103 @@ describe('acquireConcurrent (imperative)', () => {
     });
 });
 
+describe('claimCountedRateLimit', () => {
+    const uniqueKey = () =>
+        `counted-${Math.random().toString(36).slice(2, 10)}`;
+    const WINDOW = 10_000;
+    /** The start of a fixed window, so nothing before it carries in. */
+    const windowStart = (n) => vi.setSystemTime(n * WINDOW);
+
+    afterEach(() => {
+        vi.useRealTimers();
+    });
+
+    describe.each([
+        ['memory', () => configureRateLimit()],
+        [
+            'redis',
+            () =>
+                configureRateLimit({
+                    default: 'redis',
+                    redis: new RedisMock(),
+                }),
+        ],
+    ])('%s backend', (_name, configure) => {
+        beforeEach(() => {
+            configure();
+            vi.useFakeTimers({ toFake: ['Date'] });
+            windowStart(1_000);
+        });
+
+        it('admits up to the limit and spends only what fits', async () => {
+            const key = uniqueKey();
+            expect(await claimCountedRateLimit(key, 5, WINDOW, 3)).toBe(3);
+            expect(await claimCountedRateLimit(key, 5, WINDOW, 3)).toBe(2);
+            expect(await claimCountedRateLimit(key, 5, WINDOW, 1)).toBe(0);
+        });
+
+        it('spends thousands of units in one claim', async () => {
+            const key = uniqueKey();
+            expect(
+                await claimCountedRateLimit(key, 15_000, WINDOW, 6_000),
+            ).toBe(6_000);
+            expect(
+                await claimCountedRateLimit(key, 15_000, WINDOW, 10_000),
+            ).toBe(9_000);
+        });
+
+        it('carries the previous window by how much of it is still inside', async () => {
+            const key = uniqueKey();
+            expect(await claimCountedRateLimit(key, 100, WINDOW, 100)).toBe(
+                100,
+            );
+            // A quarter into the next window, 75 of those still count.
+            vi.setSystemTime(1_001 * WINDOW + WINDOW / 4);
+            expect(await claimCountedRateLimit(key, 100, WINDOW, 50)).toBe(25);
+            // Two windows on, nothing carries.
+            windowStart(1_003);
+            expect(await claimCountedRateLimit(key, 100, WINDOW, 100)).toBe(
+                100,
+            );
+        });
+
+        it('keeps its buckets apart from claimRateLimit', async () => {
+            const key = uniqueKey();
+            expect(await claimCountedRateLimit(key, 2, WINDOW, 2)).toBe(2);
+            expect(await claimRateLimit(key, 2, WINDOW, 2)).toBe(2);
+        });
+    });
+
+    it('claims nothing for zero units', async () => {
+        const key = uniqueKey();
+        expect(await claimCountedRateLimit(key, 1, WINDOW, 0)).toBe(0);
+        expect(await claimCountedRateLimit(key, 1, WINDOW, 1)).toBe(1);
+    });
+
+    it('fails open, admitting every unit, when the backend throws', async () => {
+        const redis = new RedisMock();
+        redis.multi = () => {
+            throw new Error('redis down');
+        };
+        configureRateLimit({ default: 'redis', redis });
+        const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
+        expect(await claimCountedRateLimit(uniqueKey(), 1, WINDOW, 4)).toBe(4);
+        spy.mockRestore();
+    });
+
+    it('drops a memory counter once its windows have passed', async () => {
+        configureRateLimit();
+        vi.useFakeTimers({ toFake: ['Date'] });
+        windowStart(2_000);
+        const key = uniqueKey();
+        expect(await claimCountedRateLimit(key, 1, WINDOW, 1)).toBe(1);
+        windowStart(2_002);
+        sweepMemoryWindows();
+        windowStart(2_000);
+        expect(await claimCountedRateLimit(key, 1, WINDOW, 1)).toBe(1);
+    });
+});
+
 describe('claimRateLimit', () => {
     const uniqueKey = () => `claim-${Math.random().toString(36).slice(2, 10)}`;
 

src/backend/core/http/routeLifecycle.test.ts

@@ -290,6 +290,26 @@ describe('createRouteLifecycleMiddleware', () => {
         expect(after).toHaveLength(0);
     });
 
+    it('answers 429 when the vetoing listener asks for it', async () => {
+        const events = makeEvents();
+        events.on(`${BASE}.before` as never, (_k, data) => {
+            (data as RouteLifecycleEvent).allow = false;
+            (data as RouteLifecycleEvent).rejectReason = 'slow down';
+            (data as RouteLifecycleEvent).rejectStatus = 429;
+        });
+        const reject = record(events, `${BASE}.reject` as keyof EventMap);
+
+        const mw = createRouteLifecycleMiddleware(events, POST, PATH);
+        const res = new FakeRes();
+        await mw(makeReq('u-9'), res as unknown as Response, () => {});
+
+        expect(res.statusCode).toBe(429);
+        expect(res.body).toMatchObject({
+            error: { code: 'too_many_requests', message: 'slow down' },
+        });
+        expect(reject[0]).toMatchObject({ statusCode: 429 });
+    });
+
     it('treats a listener-sent response as a terminal after (not a reject) with the real status', async () => {
         const events = makeEvents();
         events.on(`${BASE}.before` as never, (_k, data) => {

src/backend/core/http/routeLifecycle.ts

@@ -20,6 +20,7 @@
 import type { Request, RequestHandler } from 'express';
 import { actorUid } from '../actor';
 import type { EventClient } from '../../clients/event/EventClient';
+import type { LifecycleRejectStatus } from '../../clients/event/types';
 import type { RouteMethod } from './types';
 
 type RoutePathValue = string | RegExp | Array<string | RegExp> | undefined;
@@ -100,30 +101,33 @@ export const createRouteLifecycleMiddleware = (
             ...base,
             allow: true as boolean,
             rejectReason: undefined as string | undefined,
+            rejectStatus: undefined as LifecycleRejectStatus | undefined,
         };
         if (observeBefore) {
             await events.emitAndWait(beforeKey, beforeEvent, {});
         }
 
         // Explicit veto: a listener set `allow = false`. Emit `reject` and
-        // answer 403 — unless the listener already wrote its own response, in
-        // which case we report that status rather than a misleading 403.
+        // answer 403 (or the listener's 429) — unless the listener already
+        // wrote its own response, in which case we report that status.
         if (beforeEvent.allow === false) {
+            const status = beforeEvent.rejectStatus === 429 ? 429 : 403;
             events.emit(
                 `${keyBase}.reject`,
                 {
                     phase: 'reject',
                     ...base,
-                    statusCode: res.headersSent ? res.statusCode : 403,
+                    statusCode: res.headersSent ? res.statusCode : status,
                     durationMs: Date.now() - startedAt,
                     rejectReason: beforeEvent.rejectReason,
                 },
                 {},
             );
             if (!res.headersSent) {
-                res.status(403).json({
+                res.status(status).json({
                     error: {
-                        code: 'forbidden',
+                        code:
+                            status === 429 ? 'too_many_requests' : 'forbidden',
                         message:
                             beforeEvent.rejectReason ?? 'Blocked by policy',
                     },

src/backend/core/http/types.ts

@@ -76,6 +76,7 @@ export type AccountGateUser = Partial<
         | 'phone'
         | 'card_fingerprint'
         | 'otp_enabled'
+        | 'metadata'
     >
 > & { id?: number };
 

src/backend/core/schedules.test.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { describe, expect, it } from 'vitest';
+import { isHttpError } from './http/HttpError.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    SCHEDULE_PEER_PATH,
+    schedulesUnavailable,
+} from './schedules.js';
+
+describe('schedules contract', () => {
+    it('answers an absent schedules service with a 503 events_disabled', () => {
+        const err = schedulesUnavailable();
+        expect(isHttpError(err)).toBe(true);
+        expect(err.statusCode).toBe(503);
+        expect(err.legacyCode).toBe('events_disabled');
+        // Expected on a server without schedules, so it must not page.
+        expect(err.noAlarm).toBe(true);
+    });
+
+    it('keeps the list cap and the peer path the extension relies on', () => {
+        expect(SCHEDULE_LIST_LIMIT_CAP).toBe(200);
+        expect(SCHEDULE_PEER_PATH).toBe('/broadcast/events/schedules');
+    });
+});

src/backend/core/schedules.ts

@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * What core asks of schedules. Nothing here is an implementation: an extension
+ * registers `services.schedules`, and a server without one answers every
+ * schedule route and `notify({ sendAt })` with `events_disabled`. Called by
+ * extensions; keep these signatures stable.
+ */
+
+import type { EventsDeployOutcome } from '../controllers/events/workerDeploy.js';
+import type { AppMessage } from '../services/notification/appMessage.js';
+import type { SuspendedReason } from '../stores/events/DurableSubscriptionStore.js';
+import type { DurableSubscription } from '../stores/events/types.js';
+import type { PageResult } from '../util/pagination.js';
+import type { Actor } from './actor.js';
+import { HttpError } from './http/HttpError.js';
+
+/** Why a schedule ended. */
+export type ScheduleEndReason =
+    | 'removed'
+    | 'ended'
+    | 'app_deleted'
+    | 'uninstalled'
+    | 'permission_revoked'
+    | 'holder_gone'
+    | 'app_gone'
+    | 'orphaned'
+    | 'delivered'
+    | 'dropped';
+
+/** What happened to a set of subscription rows schedules own. */
+export type ScheduleSubscriptionChange =
+    | {
+          type: 'suspended';
+          reason: SuspendedReason;
+          /** For a reason that never lifts; defaults to that reason. */
+          endReason?: ScheduleEndReason;
+      }
+    | { type: 'resumed' }
+    | { type: 'ended'; endReason: ScheduleEndReason };
+
+/** A schedule that keeps running, one that runs once, or a delayed notification. */
+export type ScheduleViewKind = 'recurring' | 'once' | 'deferred';
+
+/**
+ * A schedule as its holder sees it. Times are epoch ms. `context` values are
+ * never returned, as for a persistent subscription: only its keys and a hash.
+ */
+export interface ScheduleView {
+    scheduleId: string;
+    kind: ScheduleViewKind;
+    appUid: string;
+    /** `null` once ended: the handler binding goes with the schedule. */
+    handlerName: string | null;
+    contextKeys: string[] | null;
+    contextHash: string | null;
+    /** Interval of an `every` schedule. */
+    every: number | null;
+    cron: string | null;
+    tz: string;
+    /** First allowed run; a one-shot's only run. */
+    startAt: number;
+    /** Last allowed run, inclusive. */
+    endsAt: number | null;
+    /** `sending` once a delayed send has started; it can't be cancelled then. */
+    state: 'active' | 'suspended' | 'sending' | 'ended';
+    suspendedReason: string | null;
+    endReason: string | null;
+    endedAt: number | null;
+    /**
+     * The slot it runs for next; `null` while suspended or sending, and once
+     * ended.
+     */
+    nextRunAt: number | null;
+    createdAt: number;
+}
+
+export interface ScheduleListRequest {
+    limit: number;
+    cursor?: string;
+    includeTotal: boolean;
+}
+
+/** The caps a holder's schedules are held to, from their plan. */
+export interface ScheduleQuota {
+    recurringPerUser: number;
+    recurringPerApp: number;
+    oneShotsPerUser: number;
+    oneShotsPerApp: number;
+    /** Worst-case runs per hour × 1000, summed over recurring rows. */
+    budgetPerUser: number;
+    budgetPerApp: number;
+}
+
+/** A caller cleared to supply `sendAt`, and when it asked for. */
+export interface DeferredSendPlan {
+    sendAt: number;
+    /** Due within 5 seconds, or past: sent now rather than stored. */
+    immediate: boolean;
+    holderUserId: number;
+    appUid: string;
+    handlerDepth: number;
+    quota: ScheduleQuota;
+}
+
+/** What a delayed app message stores until it is sent. */
+export interface DeferredAppMessage {
+    message: AppMessage;
+    /** A KV share handle naming the recipient. */
+    to?: string;
+}
+
+/**
+ * A condition on one row of another table that statements embed as `EXISTS
+ * (SELECT 1 FROM <table> WHERE <where>)`, binding `values`.
+ */
+export interface SqlGuard {
+    table: string;
+    where: string;
+    values: unknown[];
+}
+
+/** Ceiling of a schedule listing page. */
+export const SCHEDULE_LIST_LIMIT_CAP = 200;
+
+/** The signed peer route a region's schedule changes are sent to. */
+export const SCHEDULE_PEER_PATH = '/broadcast/events/schedules';
+
+export const schedulesUnavailable = (): HttpError =>
+    new HttpError(503, 'Schedules are not available on this server', {
+        legacyCode: 'events_disabled',
+        // The designed answer of a server without schedules, not a fault.
+        noAlarm: true,
+    });
+
+export interface ScheduleServiceContract {
+    /**
+     * Keep schedules in step with a change to the subscription rows they own.
+     * Never throws: the subscription change has already happened.
+     */
+    onSubscriptionsChanged(
+        rows: readonly DurableSubscription[],
+        change: ScheduleSubscriptionChange,
+    ): Promise<void>;
+    createFor(
+        actor: Actor,
+        body: Record<string, unknown>,
+    ): Promise<ScheduleView>;
+    list(
+        actor: Actor,
+        request: ScheduleListRequest,
+    ): Promise<PageResult<ScheduleView>>;
+    get(actor: Actor, scheduleId: unknown): Promise<ScheduleView>;
+    remove(actor: Actor, scheduleId: unknown): Promise<void>;
+    receivePeerChanges(body: unknown): Promise<{ applied: number }>;
+    /** Checks a caller supplying `sendAt`; `events_disabled` when off. */
+    prepareSendAt(actor: Actor, sendAt: unknown): Promise<DeferredSendPlan>;
+    createDeferredAppMessage(
+        plan: DeferredSendPlan,
+        message: DeferredAppMessage,
+    ): Promise<{ scheduleId: string; sendAt: number }>;
+    /**
+     * Runs an events worker deploy under the schedules' deploy gate; `capped`
+     * for a schedule run's. A passthrough when schedules are off.
+     */
+    gateDeploy?(
+        script: string,
+        options: { capped: boolean },
+        deploy: () => Promise<EventsDeployOutcome>,
+    ): Promise<EventsDeployOutcome>;
+}

src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts

@@ -209,6 +209,7 @@ afterAll(async () => {
 const attemptsFor = async (
     model: string,
     thrown: unknown = new Error('upstream down'),
+    extra: { provider?: string; fallback?: boolean } = {},
 ) => {
     createMock.mockRejectedValue(thrown);
     let caught: HttpError | undefined;
@@ -217,6 +218,7 @@ const attemptsFor = async (
             driver.complete({
                 model,
                 messages: [{ role: 'user', content: 'hi' }],
+                ...extra,
             }),
         );
     } catch (e) {
@@ -886,6 +888,59 @@ describe('ChatCompletionDriver duplicate-model fallback', () => {
     });
 });
 
+describe('ChatCompletionDriver fallback opt-out', () => {
+    const SHARED = 'deepseek-v4-pro';
+
+    it('fails on the first route when the caller sets fallback: false', async () => {
+        const attempts = await attemptsFor(SHARED, undefined, {
+            fallback: false,
+        });
+
+        expect(attempts).toHaveLength(1);
+        expect(attempts[0]).toMatchObject({ provider: 'deepseek' });
+    });
+
+    it('tries only the pinned provider', async () => {
+        const attempts = await attemptsFor(SHARED, undefined, {
+            provider: 'openrouter',
+            fallback: false,
+        });
+
+        expect(attempts).toHaveLength(1);
+        expect(attempts[0]).toMatchObject({ provider: 'openrouter' });
+    });
+
+    it('rejects a pinned provider that has no route for the model', async () => {
+        createMock.mockClear();
+        await expect(
+            withTestActor(() =>
+                driver.complete({
+                    model: SHARED,
+                    provider: 'claude',
+                    fallback: false,
+                    messages: [{ role: 'user', content: 'hi' }],
+                }),
+            ),
+        ).rejects.toMatchObject({ statusCode: 400 });
+        expect(createMock).not.toHaveBeenCalled();
+    });
+
+    it('still marks the failed route for the next caller', async () => {
+        const [only] = await attemptsFor(SHARED, undefined, {
+            fallback: false,
+        });
+
+        expect(isRouteUnhealthy(only.provider, only.model)).toBe(true);
+    });
+
+    it('keeps the chain when fallback is true or unset', async () => {
+        expect(
+            await attemptsFor(SHARED, undefined, { fallback: true }),
+        ).toHaveLength(3);
+        expect(await attemptsFor(SHARED)).toHaveLength(3);
+    });
+});
+
 // Each attempt in a fallback chain is a whole completion at that model's
 // prices, so each one has to clear the credit gate on its own. The chain used
 // to re-check with a nominal 1-microcent amount, which any account with a

src/backend/drivers/ai-chat/ChatCompletionDriver.ts

@@ -709,6 +709,20 @@ export class ChatCompletionDriver extends PuterDriver {
                 legacyCode: 'bad_request',
             });
         }
+        // A pinned provider without a route for the model normally resolves
+        // to another vendor; with `fallback: false` that is the one thing the
+        // caller asked us not to do.
+        if (
+            args.fallback === false &&
+            intendedProvider &&
+            model.provider !== intendedProvider
+        ) {
+            throw new HttpError(
+                400,
+                `Model ${args.model} is not served by provider ${intendedProvider}`,
+                { legacyCode: 'bad_request' },
+            );
+        }
 
         args.messages = normalizeMediaParts(normalize_messages(args.messages));
         if (args.tools) {
@@ -946,8 +960,15 @@ export class ChatCompletionDriver extends PuterDriver {
             const bucketKey = model.id;
             const tried = new Set([routeId(model.provider!, model.id)]);
             let lastError: Error | null = e as Error;
+            // With `fallback: false` the caller wants this route's answer or
+            // its error, not a bill for another provider's attempt.
+            const fallbackAllowed = args.fallback !== false;
 
-            while (lastError && attempts.length < MAX_ATTEMPTS) {
+            while (
+                fallbackAllowed &&
+                lastError &&
+                attempts.length < MAX_ATTEMPTS
+            ) {
                 const fallback = this.#findFallback(bucketKey, tried);
                 if (!fallback) break;
 

src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts

@@ -809,6 +809,11 @@ describe('Alibaba official catalog', () => {
         expect(models.some((model) => model.id === 'qwen-image-fixture')).toBe(
             false,
         );
+        const fixture = models.find(
+            (model) => model.id === 'qwen-catalog-fixture',
+        )!;
+        expect(fixture.puterId).toBe('alibaba:qwen/qwen-catalog-fixture');
+        expect(modelLookupNames(models)).toContain('qwen-catalog-fixture');
         const plus = models.find((model) => model.id === 'qwen-plus')!;
         const original = ALIBABA_MODELS.find(
             (model) => model.id === 'qwen-plus',
@@ -909,6 +914,115 @@ describe('Alibaba official catalog', () => {
         expect(model.costs).toEqual({ tokens: 1_000_000 });
     });
 
+    it.each([
+        ['qwen3.9-plus-2027-01-01', 'alibaba:qwen/qwen3.9-plus-2027-01-01'],
+        ['kimi/kimi-k3', 'alibaba:kimi/kimi-k3'],
+        [
+            'deepseek-v4.1-flash-1017',
+            'alibaba:deepseek/deepseek-v4.1-flash-1017',
+        ],
+        ['glm-5.3-prime', 'alibaba:zhipu/glm-5.3-prime'],
+        ['decision-model-preview', 'alibaba:qwen/decision-model-preview'],
+    ])(
+        'gives uncurated model %s a puterId in the alibaba:<vendor>/<id> form',
+        (id, puterId) => {
+            const model = mapAlibabaCatalogModel({
+                model: id,
+                capabilities: ['TG'],
+            })!;
+            expect(model.puterId).toBe(puterId);
+        },
+    );
+
+    it('skips a vendor-prefixed spelling of a curated model', () => {
+        const model = mapAlibabaCatalogModel({
+            model: 'zhipu/glm-5.2',
+            capabilities: ['TG', 'Reasoning'],
+            inference_metadata: {
+                request_modality: ['Text'],
+                response_modality: ['Text'],
+            },
+        });
+        expect(model).toBeNull();
+    });
+
+    it.each([
+        [
+            'ASR',
+            {
+                capabilities: ['ASR'],
+                inference_metadata: {
+                    request_modality: ['Audio'],
+                    response_modality: ['Text'],
+                },
+            },
+        ],
+        [
+            'rerank',
+            {
+                capabilities: ['TR'],
+                inference_metadata: {
+                    request_modality: ['Text'],
+                    response_modality: ['Text'],
+                },
+            },
+        ],
+        [
+            'image generation',
+            {
+                capabilities: ['IG'],
+                inference_metadata: {
+                    request_modality: ['Text'],
+                    response_modality: ['Text', 'Image'],
+                },
+            },
+        ],
+        [
+            'realtime omni',
+            {
+                capabilities: ['Realtime-Omni'],
+                inference_metadata: {
+                    request_modality: ['Text', 'Image', 'Audio', 'Video'],
+                    response_modality: ['Text', 'Audio'],
+                },
+            },
+        ],
+        [
+            'TTS',
+            {
+                capabilities: ['TTS'],
+                inference_metadata: {
+                    request_modality: ['Audio'],
+                    response_modality: ['Text'],
+                },
+            },
+        ],
+    ])('rejects %s models that cannot serve chat completions', (_kind, raw) => {
+        expect(
+            mapAlibabaCatalogModel({
+                model: 'non-chat-fixture',
+                ...(raw as AlibabaApiModel),
+            }),
+        ).toBeNull();
+    });
+
+    it('admits a vision model', () => {
+        const model = mapAlibabaCatalogModel({
+            model: 'vision-fixture',
+            capabilities: ['VU'],
+            inference_metadata: {
+                request_modality: ['Text', 'Image'],
+                response_modality: ['Text'],
+            },
+        })!;
+        expect(model.modalities?.input).toEqual(['text', 'image']);
+    });
+
+    it('does not admit an uncurated entry without capability or modality data', () => {
+        expect(mapAlibabaCatalogModel({ model: 'mystery-model' })).toBeNull();
+        expect(mapAlibabaCatalogModel({ model: 'qwen-plus' })).not.toBeNull();
+    });
+
     it('refuses inference with unverified catalog prices', async () => {
         seedPages();
         const { provider } = makeProvider();

src/backend/drivers/ai-chat/providers/alibaba/catalog.ts

@@ -82,6 +82,23 @@ const positiveLimit = (value: unknown): number | undefined =>
         ? value
         : undefined;
 
+const chatCapabilities = ['TG', 'Reasoning', 'VU'];
+
+const curatedPuterIds = new Set(ALIBABA_MODELS.map((model) => model.puterId));
+
+const inferPuterId = (id: string): string => {
+    if (id.includes('/')) return `alibaba:${id}`;
+    const family = id.split('-')[0];
+    const sibling = ALIBABA_MODELS.find(
+        (model) => model.puterId && model.id.split('-')[0] === family,
+    );
+    // Unprefixed feed ids are Model Studio's own family unless a curated
+    // sibling points at a third-party org.
+    const org =
+        sibling?.puterId?.slice('alibaba:'.length).split('/')[0] ?? 'qwen';
+    return `alibaba:${org}/${id}`;
+};
+
 export const mapAlibabaCatalogModel = (
     raw: AlibabaApiModel,
     previous?: IChatModel,
@@ -90,16 +107,25 @@ export const mapAlibabaCatalogModel = (
     const id = raw.model.trim();
     const existing =
         previous ?? ALIBABA_MODELS.find((model) => model.id === id);
+    const input = raw.inference_metadata?.request_modality;
     const output = raw.inference_metadata?.response_modality;
-    if (output && !output.includes('Text')) return null;
     if (
-        !output &&
-        !existing &&
-        !raw.capabilities?.some((capability) =>
-            ['TG', 'Reasoning', 'VU'].includes(capability),
+        raw.capabilities &&
+        !raw.capabilities.some((capability) =>
+            chatCapabilities.includes(capability),
         )
     )
         return null;
+    if (input && !input.includes('Text')) return null;
+    if (output && !output.includes('Text')) return null;
+    if (!raw.capabilities && !input && !output && !existing) return null;
+    let puterId = existing?.puterId;
+    if (!puterId) {
+        puterId = inferPuterId(id);
+        // A feed entry spelling a curated model differently would duplicate
+        // the curated entry's puterId and list the model twice.
+        if (curatedPuterIds.has(puterId)) return null;
+    }
     const catalogPricing: AlibabaCatalogTier[] = (raw.prices ?? []).map(
         (tier) => {
             // Abbreviated ranges do not specify whether k means 1000 or 1024 tokens.
@@ -149,6 +175,7 @@ export const mapAlibabaCatalogModel = (
     return {
         ...existing,
         id,
+        puterId,
         name: raw.name ?? existing?.name ?? id,
         description: raw.description ?? existing?.description,
         aliases: existing?.aliases,

src/backend/drivers/ai-chat/types.ts

@@ -175,6 +175,11 @@ export interface UsageDetails {
 export interface ICompleteArguments {
     messages: PuterMessage[];
     provider?: string;
+    /**
+     * `false` makes the first route's failure final instead of retrying the
+     * request on another provider that serves the same model.
+     */
+    fallback?: boolean;
     stream?: boolean;
     model: string;
     test_mode?: boolean;

src/backend/drivers/ai-chat/utils/usageEstimate.test.ts

@@ -39,9 +39,49 @@ describe('estimateTextTokens', () => {
         const long = estimateTextTokens('hello world '.repeat(100));
         expect(long).toBeGreaterThan(short);
     });
+
+    it('counts words the way splitting on \\s+ does', () => {
+        const splitEstimate = (text: string) =>
+            Math.floor(
+                (text.length / 4 + text.split(/\s+/).length * (4 / 3)) / 2,
+            );
+        for (const text of [
+            'a',
+            ' a',
+            'a ',
+            '  a  b\t\nc  ',
+            'x y z w 

  　v',
+            'é ü 日本語 テキスト',
+            '\r\n\v\f',
+        ]) {
+            expect(estimateTextTokens(text), JSON.stringify(text)).toBe(
+                splitEstimate(text),
+            );
+        }
+    });
+
+    it('gives text past the scan budget the word density scanned so far', () => {
+        const text = 'ab cd '.repeat(1000);
+        const budget = { chars: 12, scannedChars: 0, scannedRuns: 0 };
+        // Uniform text extrapolates to the full count.
+        expect(estimateTextTokens(text, budget)).toBe(estimateTextTokens(text));
+        expect(budget.chars).toBe(0);
+        // A later part with no budget left takes the same density.
+        expect(estimateTextTokens(text, budget)).toBe(estimateTextTokens(text));
+    });
 });
 
 describe('estimatePromptTokens', () => {
+    it('scans a bounded amount of text per prompt', () => {
+        // Many parts, each under the budget alone, past it together; the
+        // unscanned ones are extrapolated, which is exact for uniform text.
+        const part = 'lorem ipsum dolor '.repeat(30_000);
+        const parts = Array.from({ length: 8 }, () => ({ text: part }));
+        expect(estimatePromptTokens([{ role: 'user', content: parts }])).toBe(
+            8 * estimateTextTokens(part),
+        );
+    });
+
     it('counts plain string content', () => {
         const tokens = estimatePromptTokens([
             { role: 'user', content: 'a fairly ordinary sentence to price' },

src/backend/drivers/ai-chat/utils/usageEstimate.ts

@@ -29,7 +29,11 @@
  * passes every gate.
  */
 
-import { estimateTextTokens } from '../../util/tokenEstimate.js';
+import {
+    createWordScanBudget,
+    estimateTextTokens,
+    type WordScanBudget,
+} from '../../util/tokenEstimate.js';
 
 // Re-exported so existing consumers keep one import site for all estimators.
 export { estimateTextTokens } from '../../util/tokenEstimate.js';
@@ -91,23 +95,25 @@ const attachmentTokens = (value: unknown): number => {
  * recognises is charged the unmeasured estimate — the alternative is a content
  * type nobody has taught the gate about arriving for free.
  */
-const partTokens = (part: unknown): number => {
-    if (typeof part === 'string') return estimateTextTokens(part);
+const partTokens = (part: unknown, budget: WordScanBudget): number => {
+    if (typeof part === 'string') return estimateTextTokens(part, budget);
     if (!part || typeof part !== 'object') return 0;
 
     const p = part as ContentPart;
 
-    if (typeof p.text === 'string') return estimateTextTokens(p.text);
+    if (typeof p.text === 'string') return estimateTextTokens(p.text, budget);
 
     // Model-output shapes replayed as history: Anthropic thinking blocks,
     // our own streamed `reasoning` chunks, OpenAI refusals. All text — they
     // must not fall through to the attachment default below, which would
     // price a few sentences of reasoning like a full-frame image.
-    if (typeof p.thinking === 'string') return estimateTextTokens(p.thinking);
+    if (typeof p.thinking === 'string')
+        return estimateTextTokens(p.thinking, budget);
     if (typeof p.reasoning === 'string') {
-        return estimateTextTokens(p.reasoning);
+        return estimateTextTokens(p.reasoning, budget);
     }
-    if (typeof p.refusal === 'string') return estimateTextTokens(p.refusal);
+    if (typeof p.refusal === 'string')
+        return estimateTextTokens(p.refusal, budget);
 
     // OpenAI-style `{ image_url: { url } }` (or the flattened string form).
     if (p.image_url) {
@@ -136,6 +142,7 @@ const partTokens = (part: unknown): number => {
     if (p.type === 'tool_use' && p.input !== undefined) {
         return estimateTextTokens(
             typeof p.input === 'string' ? p.input : JSON.stringify(p.input),
+            budget,
         );
     }
     if (p.type === 'tool_result') {
@@ -143,10 +150,11 @@ const partTokens = (part: unknown): number => {
             typeof p.content === 'string'
                 ? p.content
                 : JSON.stringify(p.content ?? ''),
+            budget,
         );
     }
     if (p.type === 'compaction') {
-        return estimateTextTokens(String(p.encrypted_content ?? ''));
+        return estimateTextTokens(String(p.encrypted_content ?? ''), budget);
     }
 
     return UNMEASURED_ATTACHMENT_TOKENS;
@@ -164,21 +172,22 @@ const partTokens = (part: unknown): number => {
 export const estimatePromptTokens = (messages: unknown): number => {
     if (!Array.isArray(messages)) return 0;
 
+    const budget = createWordScanBudget();
     let tokens = 0;
     for (const message of messages) {
         if (typeof message === 'string') {
-            tokens += estimateTextTokens(message);
+            tokens += estimateTextTokens(message, budget);
             continue;
         }
         if (!message || typeof message !== 'object') continue;
 
         const content = (message as ContentPart).content;
         if (typeof content === 'string') {
-            tokens += estimateTextTokens(content);
+            tokens += estimateTextTokens(content, budget);
         } else if (Array.isArray(content)) {
-            for (const part of content) tokens += partTokens(part);
+            for (const part of content) tokens += partTokens(part, budget);
         } else if (content) {
-            tokens += partTokens(content);
+            tokens += partTokens(content, budget);
         }
     }
     return tokens;

src/backend/drivers/apps/AppDriver.js

@@ -32,10 +32,12 @@ import {
     validateIconDataUrl,
 } from '../../util/appIcon.js';
 import { toAppReadView } from '../../util/appView.js';
+import { awaitLater } from '../../util/concurrency.js';
 import { isUniqueViolation } from '../../util/dbError.js';
 import {
     buildHostedSubdomainIndexUrlCandidates,
     extractPuterHostedSubdomain,
+    resolveCustomHostSubdomain,
 } from '../../util/hostedAppBacking.js';
 import {
     normalizeLimit,
@@ -201,30 +203,38 @@ export class AppDriver extends PuterDriver {
 
         const fields = await this.#validateInput(object, { isCreate: true });
 
-        // Puter-hosted index_url handling. Order matches v1 AppES:
-        //   1. Refuse if the index_url's subdomain isn't owned by this user.
+        // Puter-hosted index_url handling (custom domains included). Order
+        // matches v1 AppES:
+        //   1. Refuse if the index_url's site isn't owned by this user.
         //   2. Try to merge into an existing row with the same index_url
         //      (origin-bootstrap takeover, or claiming an unowned row).
         //   3. Otherwise enforce index_url uniqueness so two rows can't
         //      share a hosted URL.
+        // The name lookup runs alongside so the host checks add no round
+        // trip of their own.
+        const nameTaken = awaitLater(this.appStore.existsByName(fields.name));
+        const customSite = await this.#customSiteForIndexUrl(fields.index_url);
         await this.#ensurePuterSiteSubdomainIsOwned(
             fields.index_url,
             actor.user,
+            customSite,
         );
         const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlApp({
             object,
             options,
             user: actor.user,
+            customSite,
         });
         if (joinedApp) {
             return joinedApp;
         }
         await this.#ensureIndexUrlNotAlreadyInUse({
             indexUrl: fields.index_url,
+            customSite,
         });
 
         // Name conflict handling
-        if (await this.appStore.existsByName(fields.name)) {
+        if (await nameTaken) {
             if (options?.dedupe_name) {
                 let candidate;
                 let i = 0;
@@ -424,13 +434,23 @@ export class AppDriver extends PuterDriver {
             existing: app,
         });
 
+        // Started before the index_url checks so they add no round trip.
+        const renaming = !!fields.name && fields.name !== app.name;
+        const nameTaken = renaming
+            ? awaitLater(this.appStore.existsByName(fields.name))
+            : null;
+
         // Puter-hosted index_url handling on update — same flow as create.
         // #validateInput already dropped an unchanged index_url. Self-app is
         // excluded from the conflict search via `excludeAppId`.
         if (fields.index_url) {
+            const customSite = await this.#customSiteForIndexUrl(
+                fields.index_url,
+            );
             await this.#ensurePuterSiteSubdomainIsOwned(
                 fields.index_url,
                 actor.user,
+                customSite,
             );
             const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlApp({
                 object,
@@ -438,6 +458,7 @@ export class AppDriver extends PuterDriver {
                 user: actor.user,
                 sourceAppUid: app.uid,
                 excludeAppId: app.id,
+                customSite,
             });
             if (joinedApp) {
                 return joinedApp;
@@ -445,6 +466,7 @@ export class AppDriver extends PuterDriver {
             await this.#ensureIndexUrlNotAlreadyInUse({
                 indexUrl: fields.index_url,
                 excludeAppId: app.id,
+                customSite,
             });
             if (
                 app.is_private &&
@@ -465,14 +487,10 @@ export class AppDriver extends PuterDriver {
         }
 
         // Name conflict check (only if name is changing)
-        if (fields.name && fields.name !== app.name) {
-            if (await this.appStore.existsByName(fields.name)) {
-                throw new HttpError(
-                    409,
-                    'An app with this name already exists',
-                    { legacyCode: 'conflict' },
-                );
-            }
+        if (nameTaken && (await nameTaken)) {
+            throw new HttpError(409, 'An app with this name already exists', {
+                legacyCode: 'conflict',
+            });
         }
 
         const filetypes = fields.filetype_associations;
@@ -863,6 +881,26 @@ export class AppDriver extends PuterDriver {
         return !!this.#extractPuterHostedSubdomain(indexUrl);
     }
 
+    /**
+     * `{ host, subdomain }` when `indexUrl` is on a custom domain that serves
+     * one of our sites, else null. Such a host gets the hosted-subdomain rules:
+     * owner-only, one app per URL, bootstrap rows merged.
+     */
+    async #customSiteForIndexUrl(indexUrl) {
+        if (!indexUrl || this.#isPuterHostedIndexUrl(indexUrl)) return null;
+        const parsed = URL.parse(indexUrl);
+        if (parsed?.protocol !== 'https:' && parsed?.protocol !== 'http:') {
+            return null;
+        }
+        const host = parsed.hostname.toLowerCase().replace(/\.+$/, '');
+        const subdomain = await resolveCustomHostSubdomain(
+            host,
+            this.config,
+            this.clients.event,
+        );
+        return subdomain ? { host, subdomain } : null;
+    }
+
     /**
      * Read normalized origin-alias groups from config. Each group is a deduped
      * list of lowercased, trimmed bare hosts. Malformed entries are skipped so
@@ -942,9 +980,18 @@ export class AppDriver extends PuterDriver {
         }
     }
 
-    async #findIndexUrlConflictRow({ indexUrl, excludeAppId } = {}) {
+    async #findIndexUrlConflictRow({
+        indexUrl,
+        excludeAppId,
+        customSite,
+    } = {}) {
         const aliasGroup = this.#findOriginAliasGroupForIndexUrl(indexUrl);
-        if (!this.#isPuterHostedIndexUrl(indexUrl) && !aliasGroup) return null;
+        if (
+            !this.#isPuterHostedIndexUrl(indexUrl) &&
+            !aliasGroup &&
+            !customSite
+        )
+            return null;
 
         const candidates = new Set(
             this.#buildEquivalentIndexUrlCandidates(indexUrl),
@@ -963,15 +1010,18 @@ export class AppDriver extends PuterDriver {
 
         // For alias-group hosts, treat the group as a host-level reservation:
         // any row whose index_url is the root URL of any group member counts
-        // as a conflict, so a single app owns the whole group.
-        if (aliasGroup) {
-            for (const host of aliasGroup) {
-                for (const proto of ['https', 'http']) {
-                    const base = `${proto}://${host}`;
-                    candidates.add(base);
-                    candidates.add(`${base}/`);
-                    candidates.add(`${base}/index.html`);
-                }
+        // as a conflict, so a single app owns the whole group. A custom
+        // domain's root is reserved the same way.
+        const reservedHosts = [
+            ...(aliasGroup ?? []),
+            ...(customSite ? [customSite.host] : []),
+        ];
+        for (const host of reservedHosts) {
+            for (const proto of ['https', 'http']) {
+                const base = `${proto}://${host}`;
+                candidates.add(base);
+                candidates.add(`${base}/`);
+                candidates.add(`${base}/index.html`);
             }
         }
 
@@ -984,10 +1034,15 @@ export class AppDriver extends PuterDriver {
         });
     }
 
-    async #ensureIndexUrlNotAlreadyInUse({ indexUrl, excludeAppId } = {}) {
+    async #ensureIndexUrlNotAlreadyInUse({
+        indexUrl,
+        excludeAppId,
+        customSite,
+    } = {}) {
         const conflictRow = await this.#findIndexUrlConflictRow({
             indexUrl,
             excludeAppId,
+            customSite,
         });
         if (conflictRow) {
             throw new HttpError(400, 'App index_url already in use', {
@@ -1000,9 +1055,11 @@ export class AppDriver extends PuterDriver {
         }
     }
 
-    async #ensurePuterSiteSubdomainIsOwned(indexUrl, user) {
+    async #ensurePuterSiteSubdomainIsOwned(indexUrl, user, customSite) {
         if (!user) return;
-        const subdomain = this.#extractPuterHostedSubdomain(indexUrl);
+        const subdomain =
+            customSite?.subdomain ??
+            this.#extractPuterHostedSubdomain(indexUrl);
         if (!subdomain) return;
 
         let row = await this.stores.subdomain.getBySubdomain(subdomain);
@@ -1015,10 +1072,19 @@ export class AppDriver extends PuterDriver {
             });
         }
         if (!row || row.user_id !== user.id) {
-            throw new HttpError(400, 'Subdomain not owned by user', {
-                legacyCode: 'subdomain_not_owned',
-                fields: { subdomain },
-            });
+            // A custom domain's site name isn't the caller's to learn.
+            throw new HttpError(
+                400,
+                customSite
+                    ? 'Domain not owned by user'
+                    : 'Subdomain not owned by user',
+                {
+                    legacyCode: 'subdomain_not_owned',
+                    fields: customSite
+                        ? { hostname: customSite.host }
+                        : { subdomain },
+                },
+            );
         }
     }
 
@@ -1131,17 +1197,19 @@ export class AppDriver extends PuterDriver {
         user,
         sourceAppUid,
         excludeAppId,
+        customSite,
     } = {}) {
         const indexUrl = object?.index_url;
-        // Alias-group hosts (`app_origin_aliases`) get the same merge
-        // treatment as puter-hosted subdomains. Without this, a bootstrap
-        // stub on a custom domain could never be absorbed — and since
-        // `#findIndexUrlConflictRow` *does* honor alias groups, the
-        // uniqueness check below would hard-reject the owner's own
-        // create/update instead of merging into the stub.
+        // Alias-group hosts (`app_origin_aliases`) and custom domains get the
+        // same merge treatment as puter-hosted subdomains. Without this, a
+        // bootstrap stub on such a host could never be absorbed — and since
+        // `#findIndexUrlConflictRow` *does* honor them, the uniqueness check
+        // below would hard-reject the owner's own create/update instead of
+        // merging into the stub.
         if (
             !this.#isPuterHostedIndexUrl(indexUrl) &&
-            !this.#findOriginAliasGroupForIndexUrl(indexUrl)
+            !this.#findOriginAliasGroupForIndexUrl(indexUrl) &&
+            !customSite
         ) {
             return null;
         }
@@ -1149,6 +1217,7 @@ export class AppDriver extends PuterDriver {
         const conflictRow = await this.#findIndexUrlConflictRow({
             indexUrl,
             excludeAppId,
+            customSite,
         });
         if (!conflictRow) return null;
 

src/backend/drivers/apps/AppDriver.test.ts

@@ -1912,6 +1912,188 @@ describe('AppDriver alias-group index_url merge', () => {
     });
 });
 
+// ── custom-domain index_url rules ───────────────────────────────────
+//
+// A host a `site.resolveHost` listener maps to a site gets the hosted
+// rules: only the site owner may point an app at it, one app per URL, and
+// creating there absorbs the origin-bootstrap row.
+
+describe('AppDriver custom-domain index_url', () => {
+    const hosts = new Map<string, string>();
+    let failResolve = false;
+    const resolveHost = (
+        _key: unknown,
+        data: { host: string; result: { subdomain: string | null } },
+    ) => {
+        if (failResolve) throw new Error('resolver down');
+        const sub = hosts.get(data.host);
+        if (sub) data.result.subdomain = sub;
+    };
+    const driverConfig = () =>
+        (driver as unknown as { config: Record<string, unknown> }).config;
+
+    beforeAll(() => {
+        driverConfig().custom_domains_enabled = true;
+        server.clients.event.on('site.resolveHost', resolveHost);
+    });
+
+    afterAll(() => {
+        delete driverConfig().custom_domains_enabled;
+        server.clients.event.off('site.resolveHost', resolveHost);
+    });
+
+    const attachDomain = async (userId: number) => {
+        const subdomain = uniqueName('cd');
+        await server.stores.subdomain.create({ userId, subdomain });
+        const host = `${uniqueName('www')}.custom-domain.test`;
+        hosts.set(host, subdomain);
+        return { host, subdomain };
+    };
+
+    it("refuses an index_url on another user's custom domain", async () => {
+        const owner = await makeUser();
+        const other = await makeUser();
+        const { host, subdomain } = await attachDomain(owner.userId);
+
+        const err = await withActor(other.actor, () =>
+            driver.create({
+                object: {
+                    name: uniqueName('cd-squat'),
+                    title: 't',
+                    index_url: `https://${host}/`,
+                },
+            }),
+        ).catch((e: unknown) => e);
+        expect(err).toMatchObject({
+            statusCode: 400,
+            legacyCode: 'subdomain_not_owned',
+        });
+        expect(JSON.stringify(err)).not.toContain(subdomain);
+    });
+
+    it("refuses repointing an app onto another user's custom domain", async () => {
+        const owner = await makeUser();
+        const other = await makeUser();
+        const { host } = await attachDomain(owner.userId);
+        const created = await withActor(other.actor, () =>
+            driver.create({
+                object: {
+                    name: uniqueName('cd-repoint'),
+                    title: 't',
+                    index_url: uniqueIndexUrl(),
+                },
+            }),
+        );
+
+        await expect(
+            withActor(other.actor, () =>
+                driver.update({
+                    uid: created.uid,
+                    object: { index_url: `http://${host}/sub/` },
+                }),
+            ),
+        ).rejects.toMatchObject({
+            statusCode: 400,
+            legacyCode: 'subdomain_not_owned',
+        });
+    });
+
+    it("absorbs the owner's bootstrap row on create", async () => {
+        const { actor, userId } = await makeUser();
+        const { host } = await attachDomain(userId);
+        // AuthController stamps the site owner on the bootstrap row.
+        const stubUid = `app-${uuidv4()}`;
+        await server.stores.app.createFromOrigin(stubUid, `https://${host}`, {
+            ownerUserId: userId,
+        });
+        const name = uniqueName('cd-create');
+
+        const result = await withActor(actor, () =>
+            driver.create({
+                object: { name, title: 'Mine', index_url: `https://${host}/` },
+            }),
+        );
+
+        expect(result.uid).toBe(stubUid);
+        expect(result.name).toBe(name);
+        expect((await server.stores.app.getByUid(stubUid))?.owner_user_id).toBe(
+            userId,
+        );
+    });
+
+    it('keeps one app per custom-domain URL', async () => {
+        const { actor, userId } = await makeUser();
+        const { host } = await attachDomain(userId);
+        const first = await withActor(actor, () =>
+            driver.create({
+                object: {
+                    name: uniqueName('cd-first'),
+                    title: 't',
+                    index_url: `https://${host}/`,
+                },
+            }),
+        );
+
+        await expect(
+            withActor(actor, () =>
+                driver.create({
+                    object: {
+                        name: uniqueName('cd-second'),
+                        title: 't',
+                        index_url: `http://${host}/index.html`,
+                    },
+                }),
+            ),
+        ).rejects.toMatchObject({
+            statusCode: 400,
+            legacyCode: 'app_index_url_already_in_use',
+            fields: { app_uid: first.uid },
+        });
+    });
+
+    it('treats every host as external without a resolver (no prod extension)', async () => {
+        const a = await makeUser();
+        const b = await makeUser();
+        const url = uniqueIndexUrl();
+        server.clients.event.off('site.resolveHost', resolveHost);
+        try {
+            for (const { actor } of [a, b]) {
+                await withActor(actor, () =>
+                    driver.create({
+                        object: {
+                            name: uniqueName('cd-none'),
+                            title: 't',
+                            index_url: url,
+                        },
+                    }),
+                );
+            }
+        } finally {
+            server.clients.event.on('site.resolveHost', resolveHost);
+        }
+    });
+
+    it('fails closed when the resolver errors', async () => {
+        const { actor } = await makeUser();
+        failResolve = true;
+        try {
+            await expect(
+                withActor(actor, () =>
+                    driver.create({
+                        object: {
+                            name: uniqueName('cd-down'),
+                            title: 't',
+                            index_url: uniqueIndexUrl(),
+                        },
+                    }),
+                ),
+            ).rejects.toMatchObject({ statusCode: 503 });
+        } finally {
+            failResolve = false;
+        }
+    });
+});
+
 // ── hosted-subdomain ownership check ────────────────────────────────
 //
 // `#ensurePuterSiteSubdomainIsOwned` gates puter-hosted index_urls on a

src/backend/drivers/kv/KVStoreDriver.handles.integration.test.ts

@@ -45,6 +45,8 @@ import {
     kvSharePermission,
     type KvShareAccess,
 } from '../../services/kvShare/kvShares.js';
+import { claimCountedRateLimit } from '../../core/http/middleware/rateLimit.js';
+import { kvNamespace } from '../../stores/systemKv/SystemKVStore.js';
 import {
     createTestUser,
     setupPuterTestEnv,
@@ -54,6 +56,7 @@ import {
 import type { IConfig } from '../../types.js';
 import { openCursor, sealCursor } from '../../util/pagination.js';
 import type { KVStoreDriver } from './KVStoreDriver.js';
+import { KV_HANDLE_NAMESPACE_BUDGET } from './limits.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 const GLOBAL = 'os-global';
@@ -243,6 +246,36 @@ afterAll(async () => {
     await env?.shutdown();
 });
 
+/**
+ * The handle budget calls into `owner`'s namespace spend from: every holder's,
+ * or with `holder`, that holder's own.
+ */
+const budgetKey = (owner: TestUser, holder?: TestUser, appUid = GLOBAL) => {
+    const namespace = `kv:handles:${kvNamespace(owner.uuid, appUid)}`;
+    return holder ? `${namespace}:holder:${holder.id}` : namespace;
+};
+
+/** A budget's counters for this window and the one before. */
+const budgetCounters = (key: string) => {
+    const index = Math.floor(Date.now() / KV_HANDLE_NAMESPACE_BUDGET.window);
+    return [index - 1, index].map((at) => `ratec:{${key}}:${at}`);
+};
+
+/** Units spent from a budget since it was last cleared. */
+const spent = async (key: string) => {
+    let total = 0;
+    for (const counter of budgetCounters(key))
+        total += Number(await env.server.clients.redis.get(counter));
+    return total;
+};
+
+const clearBudget = (key: string) =>
+    env.server.clients.redis.del(...budgetCounters(key));
+
+/** Spend all of a budget, as though other calls had. */
+const exhaust = (key: string, limit: number) =>
+    claimCountedRateLimit(key, limit, KV_HANDLE_NAMESPACE_BUDGET.window, limit);
+
 // Every mint spends the 60/min budget, and this file makes more than that.
 beforeEach(async () => {
     for (const user of [owner, guest, stranger]) {
@@ -250,7 +283,9 @@ beforeEach(async () => {
             `rate:${EVENTS_KV_HANDLE_LIMIT.scope}:${user.id}`,
         );
         await env.server.clients.redis.del(`rate:events:subscribe:${user.id}`);
+        await clearBudget(budgetKey(owner, user));
     }
+    await clearBudget(budgetKey(owner));
 });
 
 describe('a call through a handle', () => {
@@ -410,6 +445,17 @@ describe('the level a call needs', () => {
             legacyCode: 'kv_handle_access_denied',
         });
     });
+
+    it('keeps working after the owner stops a legacy share spelled in another case', async () => {
+        const tag = uuidv4().slice(0, 8);
+        const legacy = await legacyShare(`Notes-${tag}:`, 'read');
+        await ownerSet(`notes-${tag}:x:k`, 'v');
+        const { handle } = await share(`notes-${tag}:x:`, 'read');
+
+        await shares().removeKvShare(owner.actor, legacy.handle);
+
+        expect(await guestGet(handle, 'k')).toBe('v');
+    });
 });
 
 describe('a call refused before the handle is looked at', () => {
@@ -867,11 +913,12 @@ describe('listing through a handle', () => {
 });
 
 describe('a private entry in the region', () => {
-    it('is absent to the holder, uncounted, and refuses every write', async () => {
+    it('is absent to the holder, uncounted, and gives nothing away to a write', async () => {
         const prefix = region('priv');
         await ownerSet(`${prefix}open`, 1);
         await ownerSet(`${prefix}secret`, 's', { disableSharing: true });
         const { handle } = await share(prefix, 'write');
+        const { handle: readOnly } = await share(region('privro'), 'read');
 
         expect(await guestGet(handle, 'secret')).toBeNull();
         expect(await guestGet(handle, ['open', 'secret'])).toEqual([1, null]);
@@ -886,18 +933,78 @@ describe('a private entry in the region', () => {
             ),
         ).toEqual({ items: ['open'], total: 1 });
 
+        // Deleting it answers as deleting a key that isn't there.
+        expect(
+            await as(guest.actor, (d) =>
+                d.del({ key: 'secret', ...via(handle) }),
+            ),
+        ).toBe(true);
+        expect(
+            await as(guest.actor, (d) =>
+                d.del({ key: 'never-written', ...via(handle) }),
+            ),
+        ).toBe(true);
+        expect(
+            await as(guest.actor, (d) =>
+                d.remove({ key: 'secret', paths: [''], ...via(handle) }),
+            ),
+        ).toBeNull();
+
+        // Anything else is refused as a call the handle doesn't allow.
+        let levelRefusal: unknown;
+        try {
+            await guestSet(readOnly, 'k', 1);
+        } catch (err) {
+            levelRefusal = err;
+        }
+        const { statusCode, legacyCode, message } = levelRefusal as Record<
+            string,
+            unknown
+        >;
+        expect(legacyCode).toBe('kv_handle_access_denied');
         for (const call of [
             (d: KVStoreDriver) =>
                 d.set({ key: 'secret', value: 'x', ...via(handle) }),
-            (d: KVStoreDriver) => d.del({ key: 'secret', ...via(handle) }),
+            (d: KVStoreDriver) =>
+                d.set({
+                    key: 'secret',
+                    value: 'x',
+                    ...via(handle, { onlyIfAbsent: true }),
+                }),
             (d: KVStoreDriver) =>
                 d.update({
                     key: 'secret',
                     pathAndValueMap: { a: 1 },
                     ...via(handle),
                 }),
+            (d: KVStoreDriver) =>
+                d.add({
+                    key: 'secret',
+                    pathAndValueMap: { list: 1 },
+                    ...via(handle),
+                }),
+            (d: KVStoreDriver) =>
+                d.remove({ key: 'secret', paths: ['a'], ...via(handle) }),
+            (d: KVStoreDriver) =>
+                d.incr({
+                    key: 'secret',
+                    pathAndAmountMap: { n: 1 },
+                    ...via(handle),
+                }),
+            (d: KVStoreDriver) =>
+                d.decr({
+                    key: 'secret',
+                    pathAndAmountMap: { n: 1 },
+                    ...via(handle),
+                }),
             (d: KVStoreDriver) =>
                 d.expire({ key: 'secret', ttl: 60, ...via(handle) }),
+            (d: KVStoreDriver) =>
+                d.expireAt({
+                    key: 'secret',
+                    timestamp: later(),
+                    ...via(handle),
+                }),
             (d: KVStoreDriver) =>
                 d.batchPut({
                     items: [
@@ -908,18 +1015,23 @@ describe('a private entry in the region', () => {
                 }),
         ])
             await expect(as(guest.actor, call)).rejects.toMatchObject({
-                statusCode: 403,
+                statusCode,
+                legacyCode,
+                message,
             });
+
+        // A live key that isn't private still refuses `onlyIfAbsent` quietly.
         expect(
             await as(guest.actor, (d) =>
                 d.set({
-                    key: 'secret',
+                    key: 'open',
                     value: 'x',
                     ...via(handle, { onlyIfAbsent: true }),
                 }),
             ),
         ).toBe(false);
         expect(await ownerGet(`${prefix}secret`)).toBe('s');
+        expect(await ownerGet(`${prefix}open`)).toBe(1);
     });
 });
 
@@ -1175,6 +1287,44 @@ describe('what a write publishes', () => {
         });
     });
 
+    it('publishes a holder`s delete of a missing key, and nothing for a private one', async () => {
+        const prefix = region('pdel');
+        await ownerSet(`${prefix}secret`, 's', { disableSharing: true });
+        const { handle } = await share(prefix, 'write');
+        const ownerSub = (
+            await events().subscribe(owner.actor, SOCKET, {
+                subject: `kv:${GLOBAL}:${prefix}*`,
+            })
+        ).sub;
+        const guestSub = (
+            await events().subscribe(guest.actor, SOCKET, {
+                subject: `kv:${handle}:*`,
+            })
+        ).sub;
+        delivered.length = 0;
+
+        await as(guest.actor, (d) => d.del({ key: 'secret', ...via(handle) }));
+        await as(guest.actor, (d) => d.del({ key: 'missing', ...via(handle) }));
+
+        expect(await deliveryFor(guestSub.subId)).toMatchObject({
+            key: 'missing',
+        });
+        expect(await deliveryFor(ownerSub.subId)).toMatchObject({
+            key: `${prefix}missing`,
+        });
+        await new Promise((resolve) =>
+            setTimeout(resolve, EVENTS_COALESCE_WINDOW_MS * 2),
+        );
+        const keys = delivered
+            .filter((one) =>
+                [ownerSub.subId, guestSub.subId].includes(one.subId),
+            )
+            .map((one) => (one.event as { key?: string }).key);
+        expect(keys).not.toContain('secret');
+        expect(keys).not.toContain(`${prefix}secret`);
+        expect(await ownerGet(`${prefix}secret`)).toBe('s');
+    });
+
     it('marks the owner`s own writes as theirs, and a worker`s or handler`s as via a worker', async () => {
         const prefix = region('via');
         const sub = (
@@ -1297,6 +1447,157 @@ describe('what a call costs', () => {
     });
 });
 
+describe('the handle budget of a namespace', () => {
+    const { limit, perHolder } = KV_HANDLE_NAMESPACE_BUDGET;
+
+    it('is spent by every holder together, and refuses them all once it is gone', async () => {
+        const prefix = region('budget');
+        await ownerSet(`${prefix}k`, 'v');
+        const forGuest = await share(prefix, 'read');
+        const forStranger = await share(prefix, 'read', { holder: stranger });
+
+        expect(await guestGet(forGuest.handle, 'k')).toBe('v');
+        expect(await guestGet(forStranger.handle, 'k', stranger.actor)).toBe(
+            'v',
+        );
+        expect(await spent(budgetKey(owner))).toBe(2);
+        expect(await spent(budgetKey(owner, guest))).toBe(1);
+
+        await exhaust(budgetKey(owner), limit);
+        for (const call of [
+            () => guestGet(forGuest.handle, 'k'),
+            () => guestGet(forStranger.handle, 'k', stranger.actor),
+        ])
+            await expect(call()).rejects.toMatchObject({
+                statusCode: 429,
+                legacyCode: 'too_many_requests',
+            });
+        // The owner's own calls spend none of it, with a handle or without.
+        expect(await ownerGet(`${prefix}k`)).toBe('v');
+        expect(await guestGet(forGuest.handle, 'k', owner.actor)).toBe('v');
+        // Another owner's namespace has its own.
+        const { handle: elsewhere } = await shares().mintKvHandle(
+            guest.actor,
+            {
+                prefix: region('budget-other'),
+                recipient: { username: owner.username },
+                access: 'read',
+            },
+            { defaultAccess: 'read' },
+        );
+        expect(await guestGet(elsewhere, 'k', owner.actor)).toBeNull();
+
+        await clearBudget(budgetKey(owner));
+        expect(await guestGet(forGuest.handle, 'k')).toBe('v');
+    });
+
+    it('keeps one holder to its share, while the others get through', async () => {
+        const prefix = region('fair');
+        await ownerSet(`${prefix}k`, 'v');
+        const forGuest = await share(prefix, 'read');
+        const forStranger = await share(prefix, 'read', { holder: stranger });
+        expect(perHolder).toBeLessThan(limit);
+
+        await exhaust(budgetKey(owner, guest), perHolder);
+        await expect(guestGet(forGuest.handle, 'k')).rejects.toMatchObject({
+            statusCode: 429,
+            legacyCode: 'too_many_requests',
+        });
+        expect(await guestGet(forStranger.handle, 'k', stranger.actor)).toBe(
+            'v',
+        );
+        // The throttled holder's refusals spent nothing of the namespace's.
+        expect(await spent(budgetKey(owner))).toBe(1);
+    });
+
+    it('spends nothing when the owner calls through a handle', async () => {
+        const prefix = region('own');
+        await ownerSet(`${prefix}k`, 'v');
+        const { handle } = await share(prefix, 'read');
+        await exhaust(budgetKey(owner, owner), perHolder);
+
+        for (let i = 0; i < 3; i++)
+            expect(await guestGet(handle, 'k', owner.actor)).toBe('v');
+        expect(await spent(budgetKey(owner))).toBe(0);
+        await exhaust(budgetKey(owner), limit);
+        expect(await guestGet(handle, 'k', owner.actor)).toBe('v');
+    });
+
+    it('refuses a throttled holder before looking at its grant', async () => {
+        const { handle } = await share(region('early'), 'read');
+        await exhaust(budgetKey(owner, guest), perHolder);
+        const checks = vi.spyOn(env.server.services.permission, 'check');
+        try {
+            await expect(guestGet(handle, 'k')).rejects.toMatchObject({
+                statusCode: 429,
+            });
+            expect(checks).not.toHaveBeenCalled();
+        } finally {
+            checks.mockRestore();
+        }
+    });
+
+    it('is only heard by a caller the grant admits', async () => {
+        const { handle } = await share(region('heard'), 'watch');
+        await exhaust(budgetKey(owner), limit);
+        // Below the level: refused for that, not for the budget.
+        await expect(guestGet(handle, 'k')).rejects.toMatchObject({
+            legacyCode: 'kv_handle_access_denied',
+        });
+    });
+
+    it('tells another app of the holder the handle is not there, however spent', async () => {
+        const app = await makeApp();
+        const otherApp = await makeApp();
+        await authorizeApp(guest, app);
+        await authorizeApp(guest, otherApp);
+        const { handle } = await share(region('apps'), 'read', {
+            appUid: app.uid,
+        });
+        const holderBudget = budgetKey(owner, guest, app.uid);
+        await clearBudget(holderBudget);
+        await exhaust(holderBudget, perHolder);
+
+        await expect(
+            guestGet(handle, 'k', appActorFor(guest, otherApp)),
+        ).rejects.toMatchObject({ legacyCode: 'kv_handle_not_found' });
+        await expect(
+            guestGet(handle, 'k', appActorFor(guest, app)),
+        ).rejects.toMatchObject({ statusCode: 429 });
+        await clearBudget(holderBudget);
+    });
+
+    it('charges a call by the storage it reads and writes', async () => {
+        const prefix = region('cost');
+        const { handle } = await share(prefix, 'write');
+        const big = 'x'.repeat(100 * 1024);
+        for (const key of ['a', 'b', 'c', 'd'])
+            await ownerSet(`${prefix}big:${key}`, big);
+        const both = async () => [
+            await spent(budgetKey(owner)),
+            await spent(budgetKey(owner, guest)),
+        ];
+        expect(await both()).toEqual([0, 0]);
+
+        await guestSet(handle, 'small', 'x');
+        await vi.waitFor(async () => expect(await both()).toEqual([3, 3]));
+        expect(await guestGet(handle, 'small')).toBe('x');
+        await vi.waitFor(async () => expect(await both()).toEqual([4, 4]));
+
+        const page = (await as(guest.actor, (d) =>
+            d.list({ as: 'entries', pattern: 'big:*', ...via(handle) }),
+        )) as unknown[];
+        expect(page).toHaveLength(4);
+        // About 400 KB read, at a unit per 8 KB.
+        await vi.waitFor(async () => {
+            for (const units of await both()) {
+                expect(units).toBeGreaterThanOrEqual(4 + 50);
+                expect(units).toBeLessThan(4 + 60);
+            }
+        });
+    });
+});
+
 describe('what a holder is ever shown', () => {
     it('names no owner, prefix or whole key in any answer, success or error', async () => {
         const prefix = region('scan');

src/backend/drivers/kv/KVStoreDriver.ts

@@ -19,12 +19,13 @@
 
 import { HttpError } from '../../core/http/HttpError.js';
 import { Context } from '../../core/context.js';
+import { claimCountedRateLimit } from '../../core/http/middleware/rateLimit.js';
 import {
     DEFAULT_FREE_SUBSCRIPTION,
     DEFAULT_TEMP_SUBSCRIPTION,
 } from '../../services/metering/consts.js';
 import { PuterDriver } from '../types.js';
-import type { Actor } from '../../core/actor.js';
+import { type Actor, isAccountContext, isAppActor } from '../../core/actor.js';
 import type { DriverConcurrentConfig, DriverRateLimitConfig } from '../meta.js';
 import {
     APP_DATA_KV_METHOD_OPS,
@@ -35,6 +36,7 @@ import {
 import {
     KV_GLOBAL_APP_KEY,
     KV_MAX_KEY_BYTES,
+    KvPrivateEntryError,
     kvNamespace,
     type KVOpts,
     type KVUsage,
@@ -55,8 +57,11 @@ import {
     sealCursor,
     unsealCursor,
 } from '../../util/pagination.js';
+import { predatesApp } from '../../util/appGeneration.js';
 import { KV_COSTS } from './costs.js';
 import {
+    KV_HANDLE_NAMESPACE_BUDGET,
+    KV_HANDLE_WRITE_UNIT_WEIGHT,
     KV_MAX_BATCH_PUT_ITEMS,
     KV_MAX_GET_KEYS,
     KV_MAX_LIST_LIMIT,
@@ -93,7 +98,20 @@ type KvCallArgs = {
  * Store options plus, for a call through a share handle, the handle and its
  * region's prefix: keys go in under it and come back out relative to it.
  */
-type DriverKvOpts = KVOpts & { handle?: string; handlePrefix?: string };
+type DriverKvOpts = KVOpts & {
+    handle?: string;
+    handlePrefix?: string;
+    /** Set when the call spends from a namespace's handle budget. */
+    handleBudget?: HandleBudget;
+};
+
+/** The budgets a holder's call through a handle spends from. */
+interface HandleBudget {
+    /** Every holder of a share into the namespace together. */
+    namespace: string;
+    /** This holder alone, within the namespace. */
+    holder: string;
+}
 
 type KvListResult =
     unknown[] | { items: unknown[]; cursor?: string; total?: number };
@@ -164,6 +182,43 @@ const handlePatternInvalid = (): HttpError =>
         { legacyCode: 'kv_handle_key_invalid' },
     );
 
+/** The holder's budget in the namespace, or the namespace's, is used up. */
+const handleBudgetSpent = (): HttpError =>
+    new HttpError(429, 'Too many requests.', {
+        legacyCode: 'too_many_requests',
+    });
+
+/** The budgets a holder's calls into `namespace` spend from. */
+const handleBudget = (
+    namespace: string,
+    holderUserId: number,
+): HandleBudget => ({
+    namespace: `kv:handles:${namespace}`,
+    holder: `kv:handles:${namespace}:holder:${holderUserId}`,
+});
+
+/** Spend `units` from a budget, and say whether any of them fit. */
+const spendHandleBudget = async (
+    key: string,
+    limit: number,
+    units: number,
+): Promise<boolean> =>
+    (await claimCountedRateLimit(
+        key,
+        limit,
+        KV_HANDLE_NAMESPACE_BUDGET.window,
+        units,
+    )) > 0;
+
+/**
+ * Whether a share's grant can answer for this actor at all: the holder's own
+ * account, or the app whose data is shared. Any other caller is refused as if
+ * the handle weren't there.
+ */
+const onShareSurface = (actor: Actor, appUid: string): boolean =>
+    isAccountContext(actor) ||
+    (isAppActor(actor) && actor.effectiveApp?.uid === appUid);
+
 const invalidCursor = (): HttpError =>
     new HttpError(400, 'invalid kv: cursor', { legacyCode: 'bad_request' });
 
@@ -331,7 +386,8 @@ export class KVStoreDriver extends PuterDriver {
 
     /**
      * Run one store call. Through a handle, an error naming a stored key names
-     * the caller's relative key instead.
+     * the caller's relative key instead, and a write refused for a private
+     * entry gets the refusal any call the handle doesn't allow gets.
      */
     async #inRegion<T>(
         opts: DriverKvOpts,
@@ -342,6 +398,7 @@ export class KVStoreDriver extends PuterDriver {
         try {
             return await call();
         } catch (err) {
+            if (err instanceof KvPrivateEntryError) throw handleAccessDenied();
             throw relativizeError(err, opts.handlePrefix, keys);
         }
     }
@@ -435,11 +492,31 @@ export class KVStoreDriver extends PuterDriver {
         if (userId !== row.granteeUserId && userId !== row.ownerUserId)
             throw handleNotFound();
 
-        // Side by side, so the app lookup costs no round trip of its own.
+        // The holder's own budget goes before the grant lookups, so throttled
+        // retries cost none. It only counts the holder's own calls, so it says
+        // nothing about the namespace. The owner spends no handle budget.
+        const { limit, perHolder } = KV_HANDLE_NAMESPACE_BUDGET;
+        const budget =
+            userId === row.ownerUserId
+                ? undefined
+                : handleBudget(kvNamespace(ownerUuid, row.appUid), userId);
+        const holderFirst =
+            budget !== undefined && onShareSurface(actor, row.appUid);
+        if (
+            holderFirst &&
+            !(await spendHandleBudget(budget.holder, perHolder, 1))
+        )
+            throw handleBudgetSpent();
+
+        // Side by side, so the app lookup costs no round trip of its own. A
+        // handle older than its app was made for an earlier app under the
+        // same uid, and is gone with it.
         const [appGone, allowed] = await Promise.all([
             row.appUid === KV_GLOBAL_APP_KEY
                 ? false
-                : this.stores.app.getByUid(row.appUid).then((app) => !app),
+                : this.stores.app
+                      .getByUid(row.appUid)
+                      .then((app) => !app || predatesApp(row.createdAt, app)),
             this.services.permission.check(
                 actor,
                 kvSharePermission(ownerUuid, row.appUid, row.keyPrefix, level),
@@ -466,6 +543,17 @@ export class KVStoreDriver extends PuterDriver {
             Buffer.byteLength(row.keyPrefix, 'utf8') > KV_SHARE_PREFIX_MAX_BYTES
         )
             throw handleAccessDenied();
+        // The namespace's budget tells how busy other holders are, so only a
+        // caller the grant admits may hear it. `#meter` spends the rest.
+        if (budget) {
+            if (
+                !holderFirst &&
+                !(await spendHandleBudget(budget.holder, perHolder, 1))
+            )
+                throw handleBudgetSpent();
+            if (!(await spendHandleBudget(budget.namespace, limit, 1)))
+                throw handleBudgetSpent();
+        }
 
         return {
             actor,
@@ -473,6 +561,7 @@ export class KVStoreDriver extends PuterDriver {
             namespaceAppUuid: row.appUid,
             handle,
             handlePrefix: row.keyPrefix,
+            ...(budget ? { handleBudget: budget } : {}),
         };
     }
 
@@ -672,7 +761,23 @@ export class KVStoreDriver extends PuterDriver {
         }
     }
 
-    #meter(actor: Actor | undefined, usage: KVUsage): void {
+    /**
+     * Bill the caller for what the call consumed. A holder's call through a
+     * handle also spends it from its budgets, past the unit admission took.
+     */
+    #meter(opts: DriverKvOpts, usage: KVUsage): void {
+        const budget = opts.handleBudget;
+        if (budget) {
+            const units = Math.ceil(
+                usage.read + usage.write * KV_HANDLE_WRITE_UNIT_WEIGHT,
+            );
+            if (units > 1) {
+                const { limit, perHolder } = KV_HANDLE_NAMESPACE_BUDGET;
+                void spendHandleBudget(budget.holder, perHolder, units - 1);
+                void spendHandleBudget(budget.namespace, limit, units - 1);
+            }
+        }
+        const actor = opts.actor;
         if (!actor) return;
         const metering = this.services.metering;
         if (usage.read > 0) {
@@ -750,7 +855,7 @@ export class KVStoreDriver extends PuterDriver {
                     opts,
                 ),
             );
-            this.#meter(opts.actor, usage);
+            this.#meter(opts, usage);
             return res;
         }
 
@@ -758,7 +863,7 @@ export class KVStoreDriver extends PuterDriver {
         const { res, usage } = await this.#inRegion(opts, [coerced], () =>
             this.stores.kv.get({ key: this.#storeKey(coerced, opts) }, opts),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -793,7 +898,7 @@ export class KVStoreDriver extends PuterDriver {
                 opts,
             ),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -859,7 +964,7 @@ export class KVStoreDriver extends PuterDriver {
                     opts,
                 ),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -872,7 +977,7 @@ export class KVStoreDriver extends PuterDriver {
         const { res, usage } = await this.#inRegion(opts, [coerced], () =>
             this.stores.kv.del({ key: this.#storeKey(coerced, opts) }, opts),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -928,7 +1033,7 @@ export class KVStoreDriver extends PuterDriver {
             },
             opts,
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         if (!listing) return res;
         return this.#foreignResult(res as KvListResult, args.as, listing);
     }
@@ -936,7 +1041,7 @@ export class KVStoreDriver extends PuterDriver {
     async flush(args: { optConfig?: KvOptConfig }): Promise<boolean> {
         const opts = await this.#opts('flush', args);
         const { res, usage } = await this.stores.kv.flush(opts);
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -964,7 +1069,7 @@ export class KVStoreDriver extends PuterDriver {
                 opts,
             ),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -992,7 +1097,7 @@ export class KVStoreDriver extends PuterDriver {
                 opts,
             ),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -1017,7 +1122,7 @@ export class KVStoreDriver extends PuterDriver {
                 opts,
             ),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -1039,7 +1144,7 @@ export class KVStoreDriver extends PuterDriver {
                 opts,
             ),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -1066,7 +1171,7 @@ export class KVStoreDriver extends PuterDriver {
                 opts,
             ),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -1091,7 +1196,7 @@ export class KVStoreDriver extends PuterDriver {
                 opts,
             ),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         return res;
     }
 
@@ -1113,7 +1218,7 @@ export class KVStoreDriver extends PuterDriver {
                 opts,
             ),
         );
-        this.#meter(opts.actor, usage);
+        this.#meter(opts, usage);
         // The result is the entry's value, so a delete-only grant gets none. A
         // handle's `write` level reads too.
         if (

src/backend/drivers/kv/limits.ts

@@ -17,8 +17,8 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-// Per-call caps on the public KV driver. Enforced here rather than in the
-// store, whose internal callers batch past them.
+// Caps on the public KV driver. Enforced in the driver rather than the store,
+// whose internal callers batch past them.
 
 /** Keys one array `get` may name. */
 export const KV_MAX_GET_KEYS = 1000;
@@ -31,3 +31,23 @@ export const KV_MAX_BATCH_PUT_ITEMS = 1000;
  * the cursor carries the rest. Matches the SDK's own page size.
  */
 export const KV_MAX_LIST_LIMIT = 1000;
+
+/**
+ * Storage capacity calls through share handles may spend in one owner's
+ * namespace per window: `limit` for every holder together, and `perHolder` for
+ * any one of them, so no holder can starve the others. The owner's own calls
+ * through a handle spend none. Sized to leave about half of what one namespace
+ * can serve to the owner's own app.
+ */
+export const KV_HANDLE_NAMESPACE_BUDGET = {
+    limit: 15_000,
+    perHolder: 3_750,
+    window: 10_000,
+};
+
+/**
+ * What a call spends from those budgets: its read units, plus this many per
+ * write unit, since a write costs the store about three times what a read does.
+ * Every call spends at least one.
+ */
+export const KV_HANDLE_WRITE_UNIT_WEIGHT = 3;

src/backend/drivers/notification/NotificationDriver.appMessage.test.ts

@@ -145,6 +145,18 @@ const captureBus = async (
     return seen;
 };
 
+const DAY_MS = 24 * 60 * 60_000;
+
+/** Exactly `count` slots of a window are spent. */
+const expectSpent = async (
+    key: string,
+    windowMs: number,
+    count: number,
+): Promise<void> => {
+    expect(await peekRateLimit(key, count, windowMs)).toBe(false);
+    expect(await peekRateLimit(key, count + 1, windowMs)).toBe(true);
+};
+
 const bootServer = (config: Partial<IConfig>) => {
     const state = {
         server: null as unknown as PuterServer,
@@ -171,16 +183,15 @@ describe('app.message with live delivery over the socket wire', () => {
         const app = await makeApp(state.server, holder.user.id!);
 
         let created: Record<string, unknown> | null = null;
-        const pushed = await captureBus(
-            state.server,
-            'outer.gui.notif.message',
-            async () => {
+        let desktop: Array<Record<string, unknown>> = [];
+        const written = await captureBus(state.server, 'notif.written', () =>
+            captureBus(state.server, 'outer.gui.notif.message', async () => {
                 created = (await withActor(asApp(holder, app), () =>
                     state.driver.create(
                         message({ text: 'Body', channel: 'replies' }),
                     ),
                 )) as Record<string, unknown>;
-            },
+            }).then((seen) => (desktop = seen)),
         );
 
         const uid = created!.uid as string;
@@ -202,13 +213,16 @@ describe('app.message with live delivery over the socket wire', () => {
             audience: 'app-user',
             type: 'app.message',
         });
-        // Not silent: it went out on the wire the desktop listens to.
-        expect(pushed).toEqual([
-            {
-                user_id_list: [holder.user.id],
-                response: { uid, notification: created!.value },
-            },
+        // Not silent, but sent for the app alone: the desktop wire stays quiet.
+        expect(written).toEqual([
+            expect.objectContaining({
+                userId: holder.user.id,
+                uid,
+                type: 'app.message',
+                appUid: app.uid,
+            }),
         ]);
+        expect(desktop).toEqual([]);
     });
 
     it("lets the app's worker and the tokens it issued notify their user", async () => {
@@ -300,6 +314,42 @@ describe('app.message with live delivery over the socket wire', () => {
         ).toEqual([]);
     });
 
+    it('names the field over its own limit before the total size', async () => {
+        const holder = await makeUser(state.server);
+        const app = await makeApp(state.server, holder.user.id!);
+        for (const [fields, field] of [
+            [{ data: { blob: 'x'.repeat(20_000) } }, 'data'],
+            [{ text: 'x'.repeat(20_000) }, 'text'],
+            [{ extra: 'x'.repeat(20_000) }, 'extra'],
+        ] as const) {
+            await expect(
+                withActor(asApp(holder, app), () =>
+                    state.driver.create(message(fields)),
+                ),
+            ).rejects.toMatchObject({
+                statusCode: 400,
+                legacyCode: 'bad_request',
+                fields: { field },
+            });
+        }
+    });
+
+    it('still caps the whole value when every field passes', async () => {
+        const holder = await makeUser(state.server);
+        const app = await makeApp(state.server, holder.user.id!);
+        // Invisible marks are dropped from the title, so only the total is over.
+        await expect(
+            withActor(asApp(holder, app), () =>
+                state.driver.create(
+                    message({ title: `${'\u200b'.repeat(20_000)}Hi` }),
+                ),
+            ),
+        ).rejects.toMatchObject({
+            statusCode: 400,
+            legacyCode: 'notification_value_too_large',
+        });
+    });
+
     it('keeps the holder creating other types silently', async () => {
         const holder = await makeUser(state.server);
         const pushed = await captureBus(
@@ -465,6 +515,176 @@ describe('app.message with live delivery over the socket wire', () => {
     });
 });
 
+describe('app.message groups', () => {
+    const state = bootServer({
+        notificationAppMessageLimits: { perMinute: 5, perDay: 4 },
+    });
+
+    const setup = async () => {
+        const holder = await makeUser(state.server);
+        const app = await makeApp(state.server, holder.user.id!);
+        const send = async (fields: Record<string, unknown> = {}) =>
+            (await withActor(asApp(holder, app), () =>
+                state.driver.create(message(fields)),
+            )) as Record<string, unknown>;
+        const rows = () =>
+            state.server.stores.notification.listByUserId(holder.user.id!, {});
+        return {
+            holder,
+            app,
+            send,
+            rows,
+            minuteKey: `notif:app-message:minute:${holder.user.id}:${app.uid}`,
+            dayKey: `notif:app-message:day:${holder.user.id}:${app.uid}`,
+        };
+    };
+
+    it('replaces the unseen message with the same group, spending no day slot', async () => {
+        const { holder, app, send, rows, minuteKey, dayKey } = await setup();
+        const first = await send({ title: 'One', group: 'chat-1' });
+
+        let second: Record<string, unknown> = {};
+        const written = await captureBus(
+            state.server,
+            'notif.written',
+            async () => {
+                second = await send({
+                    title: 'Two',
+                    text: 'Hi',
+                    group: 'chat-1',
+                });
+            },
+        );
+
+        expect(second.uid).toBe(first.uid);
+        expect(second.value).toMatchObject({ title: 'Two', text: 'Hi' });
+        expect(await rows()).toEqual([
+            expect.objectContaining({
+                uid: first.uid,
+                value: expect.objectContaining({ title: 'Two' }),
+            }),
+        ]);
+        expect(written).toEqual([
+            expect.objectContaining({
+                userId: holder.user.id,
+                uid: first.uid,
+                appUid: app.uid,
+                updated: true,
+            }),
+        ]);
+        await expectSpent(minuteKey, 60_000, 2);
+        await expectSpent(dayKey, DAY_MS, 1);
+    });
+
+    it('starts a new message once the last one was shown or dismissed', async () => {
+        const { holder, app, send, rows, dayKey } = await setup();
+        const asTheApp = asApp(holder, app);
+
+        const first = await send({ group: 'chat-1' });
+        await withActor(asTheApp, () =>
+            state.driver.mark_shown({ uid: first.uid }),
+        );
+        const second = await send({ group: 'chat-1' });
+        expect(second.uid).not.toBe(first.uid);
+
+        await withActor(asTheApp, () =>
+            state.driver.mark_acknowledged({ uid: second.uid }),
+        );
+        const third = await send({ group: 'chat-1' });
+        expect(third.uid).not.toBe(second.uid);
+
+        expect(await rows()).toHaveLength(3);
+        await expectSpent(dayKey, DAY_MS, 3);
+    });
+
+    it('keeps messages with no group, or another group, apart', async () => {
+        const { send, rows } = await setup();
+        const uids = [
+            (await send()).uid,
+            (await send()).uid,
+            (await send({ group: 'a' })).uid,
+            (await send({ group: 'b' })).uid,
+        ];
+        expect(new Set(uids).size).toBe(4);
+        expect(await rows()).toHaveLength(4);
+    });
+
+    it('still spends the minute window on a replacement', async () => {
+        const { send, rows, dayKey } = await setup();
+        for (let i = 0; i < 5; i++) await send({ group: 'chat-1' });
+        await expect(send({ group: 'chat-1' })).rejects.toMatchObject({
+            statusCode: 429,
+            legacyCode: 'too_many_requests',
+        });
+        expect(await rows()).toHaveLength(1);
+        await expectSpent(dayKey, DAY_MS, 1);
+    });
+
+    it('replaces with the day window spent', async () => {
+        const { send, rows } = await setup();
+        const grouped = await send({ group: 'chat-1' });
+        for (let i = 0; i < 3; i++) await send();
+        await expect(send()).rejects.toMatchObject({ statusCode: 429 });
+
+        const replaced = await send({ title: 'Later', group: 'chat-1' });
+        expect(replaced.uid).toBe(grouped.uid);
+        expect(await rows()).toHaveLength(4);
+    });
+
+    it('writes a new message when the old one is shown before the replace lands', async () => {
+        const { holder, app, send, rows, minuteKey, dayKey } = await setup();
+        const first = await send({ group: 'chat-1' });
+
+        charges.held.add(minuteKey);
+        const pending = send({ title: 'Raced', group: 'chat-1' });
+        let second: Record<string, unknown>;
+        try {
+            await until(() => charges.waiting.length === 1);
+            await withActor(asApp(holder, app), () =>
+                state.driver.mark_shown({ uid: first.uid }),
+            );
+        } finally {
+            charges.held.clear();
+            charges.waiting.shift()?.();
+            second = await pending;
+        }
+
+        expect(second.uid).not.toBe(first.uid);
+        expect(await rows()).toHaveLength(2);
+        await expectSpent(dayKey, DAY_MS, 2);
+    });
+
+    it('folds concurrent messages to an empty group into one, spending one day slot', async () => {
+        const { send, rows, minuteKey, dayKey } = await setup();
+        const sent = await Promise.all(
+            Array.from({ length: 4 }, (_, i) =>
+                send({ title: `Message ${i}`, group: 'chat-1' }),
+            ),
+        );
+
+        expect(new Set(sent.map((s) => s.uid)).size).toBe(1);
+        expect(await rows()).toHaveLength(1);
+        await expectSpent(minuteKey, 60_000, 4);
+        await expectSpent(dayKey, DAY_MS, 1);
+    });
+
+    it('still sends when the group stays locked past the wait', async () => {
+        const { holder, app, send, rows, dayKey } = await setup();
+        await state.server.clients.redis.set(
+            `notif:app-message:group:${holder.user.id}:${app.uid}:chat-1`,
+            'stalled-sender',
+            'PX',
+            60_000,
+        );
+
+        const sent = await send({ group: 'chat-1' });
+        expect(await rows()).toEqual([
+            expect.objectContaining({ uid: sent.uid }),
+        ]);
+        await expectSpent(dayKey, DAY_MS, 1);
+    });
+});
+
 describe('app.message with the events fold-in on', () => {
     const state = bootServer({
         events: { enabled: true, notificationsFoldIn: true },
@@ -503,6 +723,34 @@ describe('app.message with the events fold-in on', () => {
         ).toBe(`notif:${app.uid}:app-user`);
     });
 
+    it('dispatches a replacement again under the same uid', async () => {
+        const holder = await makeUser(state.server);
+        const app = await makeApp(state.server, holder.user.id!);
+        const first = (await withActor(asApp(holder, app), () =>
+            state.driver.create(message({ group: 'g' })),
+        )) as Record<string, unknown>;
+
+        const created = await captureBus(state.server, 'notif.created', () =>
+            withActor(asApp(holder, app), () =>
+                state.driver.create(message({ title: 'Again', group: 'g' })),
+            ),
+        );
+        expect(created).toEqual([
+            expect.objectContaining({
+                uid: first.uid,
+                appUid: app.uid,
+                audience: 'app-user',
+                value: expect.objectContaining({ title: 'Again' }),
+                replaced: true,
+            }),
+        ]);
+        const row = await state.server.stores.notification.getByUid(
+            first.uid as string,
+            { userId: holder.user.id, primary: true },
+        );
+        expect(Number(row.replaced_at)).toBe(created[0].createdAt);
+    });
+
     it('limits each app per user per day, without spending the minute', async () => {
         const holder = await makeUser(state.server);
         const app = await makeApp(state.server, holder.user.id!);

src/backend/drivers/notification/NotificationDriver.handle.integration.test.ts

@@ -0,0 +1,437 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/** `app.message` sent to the other party of a KV share through its handle. */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import {
+    EVENTS_KV_HANDLE_LIMIT,
+    KV_SHARE_NOTIFY_LIMITS,
+} from '../../controllers/events/limits.js';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { runWithContext } from '../../core/context.js';
+import {
+    claimRateLimit,
+    peekRateLimit,
+} from '../../core/http/middleware/rateLimit.js';
+import {
+    createTestUser,
+    setupPuterTestEnv,
+    type PuterTestEnv,
+} from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import type { NotificationDriver } from './NotificationDriver.js';
+
+const BOOT_TIMEOUT_MS = 120_000;
+const DAY_MS = 24 * 60 * 60_000;
+
+interface TestUser {
+    actor: Actor;
+    username: string;
+    id: number;
+}
+
+let env: PuterTestEnv;
+let owner: TestUser;
+let guest: TestUser;
+let stranger: TestUser;
+
+const driver = () =>
+    env.server.drivers.notifications as unknown as NotificationDriver;
+
+const userFor = async (username: string): Promise<TestUser> => {
+    const row = await env.server.stores.user.getByUsername(username);
+    return {
+        actor: makeActor({ user: row as never }),
+        username,
+        id: row!.id!,
+    };
+};
+
+const makeApp = async () => {
+    const name = `notify-to-${uuidv4().slice(0, 8)}`;
+    return env.server.stores.app.create(
+        {
+            name,
+            title: 'Handle App',
+            index_url: `https://${name}.example.test/index.html`,
+        },
+        { ownerUserId: owner.id },
+    ) as Promise<{ uid: string; id: number }>;
+};
+
+const inApp = (user: TestUser, app: { uid: string; id: number }) =>
+    makeActor({
+        user: user.actor.user as never,
+        app: { uid: app.uid, id: app.id },
+    });
+
+/** A share of `app`'s data from the owner to the guest. */
+const share = async (app: { uid: string }, notify = true) =>
+    env.server.services.kvShare.mintKvHandle(
+        owner.actor,
+        {
+            prefix: `room-${uuidv4().slice(0, 8)}:`,
+            recipient: { username: guest.username },
+            appUid: app.uid,
+            notify,
+        },
+        { defaultAccess: 'write' },
+    );
+
+const send = (actor: Actor, fields: Record<string, unknown> = {}) =>
+    runWithContext({ actor }, () =>
+        driver().create({
+            object: { value: { type: 'app.message', title: 'Hi', ...fields } },
+        }),
+    ) as Promise<Record<string, unknown>>;
+
+const rowsOf = (user: TestUser) =>
+    env.server.stores.notification.listByUserId(user.id, {});
+
+/** The user's `app.message` rows from `app`, newest first. */
+const appMessagesOf = async (user: TestUser, app: { uid: string }) =>
+    (await rowsOf(user)).filter(
+        (row: Record<string, unknown>) =>
+            row.app_uid === app.uid && row.type === 'app.message',
+    );
+
+/** What a refusal shows the sender. */
+const refusalOf = async (sending: Promise<unknown>) => {
+    try {
+        await sending;
+    } catch (err) {
+        const { statusCode, legacyCode, message, fields } = err as Record<
+            string,
+            unknown
+        >;
+        return { statusCode, legacyCode, message, fields };
+    }
+    throw new Error('expected a refusal');
+};
+
+const tooMany = {
+    statusCode: 429,
+    legacyCode: 'too_many_requests',
+    message: 'Too many notifications.',
+};
+
+const minuteKey = (user: TestUser, appUid: string) =>
+    `notif:app-message:minute:${user.id}:${appUid}`;
+const dayKey = (user: TestUser, appUid: string) =>
+    `notif:app-message:day:${user.id}:${appUid}`;
+
+/** Exactly `count` slots of a window are spent. */
+const expectSpent = async (key: string, windowMs: number, count: number) => {
+    if (count > 0)
+        expect(await peekRateLimit(key, count, windowMs)).toBe(false);
+    expect(await peekRateLimit(key, count + 1, windowMs)).toBe(true);
+};
+
+/** Every payload the bus carried under `key` while `fn` ran. */
+const captureBus = async (
+    key: string,
+    fn: () => Promise<unknown>,
+): Promise<Array<Record<string, unknown>>> => {
+    const seen: Array<Record<string, unknown>> = [];
+    const handler = (_k: string, data: unknown) =>
+        seen.push(data as Record<string, unknown>);
+    env.server.clients.event.on(key, handler);
+    try {
+        await fn();
+        await new Promise((resolve) => setTimeout(resolve, 20));
+    } finally {
+        env.server.clients.event.off?.(key, handler);
+    }
+    return seen;
+};
+
+beforeAll(async () => {
+    env = await setupPuterTestEnv({
+        events: { enabled: true, kvHandles: true },
+        kv: { shareHandles: true },
+        unlimitedMetering: true,
+        notificationAppMessageLimits: { perMinute: 5, perDay: 4 },
+    } as IConfig);
+    const strangerName = `notify-to-${uuidv4().slice(0, 8)}`;
+    await createTestUser(env.server, {
+        username: strangerName,
+        password: 'pw-test-1234',
+    });
+    owner = await userFor(env.users.user.username);
+    guest = await userFor(env.users.other.username);
+    stranger = await userFor(strangerName);
+}, BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+    await env?.shutdown();
+});
+
+beforeEach(async () => {
+    await env.server.clients.redis.del(
+        `rate:${EVENTS_KV_HANDLE_LIMIT.scope}:${owner.id}`,
+    );
+});
+
+describe('app.message to the other party of a share', () => {
+    it('writes the owner’s message to the holder, and answers with nothing', async () => {
+        const app = await makeApp();
+        const { handle } = await share(app);
+
+        let sent: Record<string, unknown> | null = null;
+        const written = await captureBus('notif.written', async () => {
+            sent = await send(inApp(owner, app), {
+                title: 'New message',
+                to: { handle },
+            });
+        });
+
+        expect(sent).toEqual({});
+        const [row] = await appMessagesOf(guest, app);
+        expect(row).toMatchObject({
+            user_id: guest.id,
+            app_uid: app.uid,
+            type: 'app.message',
+            audience: 'app-user',
+        });
+        expect(row!.value).toEqual({
+            type: 'app.message',
+            title: 'New message',
+            priority: 'normal',
+            ttl: 3600,
+            app: { uid: app.uid, name: 'Handle App' },
+        });
+        // The push extension bills and checks consent on this recipient.
+        expect(written).toEqual([
+            expect.objectContaining({
+                userId: guest.id,
+                uid: row.uid,
+                appUid: app.uid,
+            }),
+        ]);
+    });
+
+    it('writes the holder’s message to the owner', async () => {
+        const app = await makeApp();
+        const { handle } = await share(app);
+        const before = (await rowsOf(guest)).length;
+
+        const sent = await send(inApp(guest, app), {
+            title: 'From the holder',
+            to: { handle },
+        });
+
+        expect(sent).toEqual({});
+        const [row] = await appMessagesOf(owner, app);
+        expect(row).toMatchObject({ user_id: owner.id });
+        expect(row.value).toMatchObject({ title: 'From the holder' });
+        expect(await rowsOf(guest)).toHaveLength(before);
+    });
+
+    it("charges the recipient's limits, not the sender's", async () => {
+        const app = await makeApp();
+        const { handle } = await share(app);
+
+        await send(inApp(owner, app), { to: { handle } });
+
+        await expectSpent(minuteKey(guest, app.uid), 60_000, 1);
+        await expectSpent(dayKey(guest, app.uid), DAY_MS, 1);
+        await expectSpent(minuteKey(owner, app.uid), 60_000, 0);
+        await expectSpent(dayKey(owner, app.uid), DAY_MS, 0);
+    });
+
+    it('answers the pair limit and the recipient’s limit the same way', async () => {
+        const app = await makeApp();
+        const { handle } = await share(app);
+
+        // The recipient's own day window runs out.
+        for (let i = 0; i < 4; i++)
+            await send(inApp(guest, app), { title: `own ${i}` });
+        const byRecipient = await refusalOf(
+            send(inApp(owner, app), { to: { handle } }),
+        );
+
+        const other = await makeApp();
+        const { handle: otherHandle } = await share(other);
+        const [minute] = KV_SHARE_NOTIFY_LIMITS;
+        await claimRateLimit(
+            `${minute.scope}:${owner.id}:${guest.id}:${other.uid}`,
+            minute.limit,
+            minute.window,
+            minute.limit,
+        );
+        const byPair = await refusalOf(
+            send(inApp(owner, other), { to: { handle: otherHandle } }),
+        );
+
+        expect(byRecipient).toEqual({ ...tooMany, fields: undefined });
+        expect(byPair).toEqual(byRecipient);
+        await expectSpent(dayKey(guest, other.uid), DAY_MS, 0);
+    });
+
+    it("replaces the recipient's newest message in the same group", async () => {
+        const app = await makeApp();
+        const { handle } = await share(app);
+
+        await send(inApp(owner, app), {
+            title: 'One',
+            group: 'room-1',
+            to: { handle },
+        });
+        const second = await send(inApp(owner, app), {
+            title: 'Two',
+            group: 'room-1',
+            to: { handle },
+        });
+
+        expect(second).toEqual({});
+        const rows = await appMessagesOf(guest, app);
+        expect(rows).toHaveLength(1);
+        expect(rows[0].value).toMatchObject({ title: 'Two' });
+        await expectSpent(dayKey(guest, app.uid), DAY_MS, 1);
+    });
+
+    it('folds concurrent messages to one group into one, through the share or not', async () => {
+        const app = await makeApp();
+        const { handle } = await share(app);
+
+        await Promise.all([
+            send(inApp(owner, app), { group: 'room-1', to: { handle } }),
+            send(inApp(owner, app), { group: 'room-1', to: { handle } }),
+            send(inApp(guest, app), { group: 'room-1' }),
+            send(inApp(guest, app), { group: 'room-1' }),
+        ]);
+
+        expect(await appMessagesOf(guest, app)).toHaveLength(1);
+        await expectSpent(dayKey(guest, app.uid), DAY_MS, 1);
+    });
+
+    it.each([
+        ['a string', 'kvh-1'],
+        ['an array', [{ handle: 'kvh-1' }]],
+        ['null', null],
+        ['no handle', {}],
+        ['a handle that is not a string', { handle: 7 }],
+        ['an empty handle', { handle: '' }],
+        ['another field beside the handle', { handle: 'kvh-1', user: 'x' }],
+    ])('refuses `to` as %s', async (_label, to) => {
+        const app = await makeApp();
+        await expect(send(inApp(owner, app), { to })).rejects.toMatchObject({
+            statusCode: 400,
+            legacyCode: 'bad_request',
+            fields: { field: 'to' },
+        });
+    });
+
+    it('validates the message before spending the pair limit', async () => {
+        const app = await makeApp();
+        const { handle } = await share(app);
+        const [minute] = KV_SHARE_NOTIFY_LIMITS;
+
+        await expect(
+            send(inApp(owner, app), {
+                icon: 'http://insecure.test/a.png',
+                to: { handle },
+            }),
+        ).rejects.toMatchObject({ statusCode: 400, fields: { field: 'icon' } });
+        expect(
+            await env.server.clients.redis.zcard(
+                `rate:${minute.scope}:${owner.id}:${guest.id}:${app.uid}`,
+            ),
+        ).toBe(0);
+    });
+
+    describe('passes every refusal of the share through with its code', () => {
+        const absent = {
+            statusCode: 404,
+            legacyCode: 'subject_does_not_exist',
+        };
+
+        it('an unknown handle, or one the sender is no party to', async () => {
+            const app = await makeApp();
+            const { handle } = await share(app);
+            await expect(
+                send(inApp(owner, app), { to: { handle: `kvh-${uuidv4()}` } }),
+            ).rejects.toMatchObject(absent);
+            await expect(
+                send(inApp(stranger, app), { to: { handle } }),
+            ).rejects.toMatchObject(absent);
+            await expect(
+                send(inApp(owner, await makeApp()), { to: { handle } }),
+            ).rejects.toMatchObject(absent);
+        });
+
+        it('a share without notify', async () => {
+            const app = await makeApp();
+            const { handle } = await share(app, false);
+            await expect(
+                send(inApp(owner, app), { to: { handle } }),
+            ).rejects.toMatchObject({
+                statusCode: 403,
+                legacyCode: 'kv_share_notify_not_enabled',
+            });
+        });
+
+        it('a recipient who blocked the sender', async () => {
+            const app = await makeApp();
+            const { handle } = await share(app);
+            await env.server.stores.userBlock.create(guest.id, owner.id);
+            try {
+                await expect(
+                    send(inApp(owner, app), { to: { handle } }),
+                ).rejects.toMatchObject({
+                    statusCode: 403,
+                    legacyCode: 'recipient_not_accepting_shares',
+                });
+            } finally {
+                await env.server.stores.userBlock.deleteByPair(
+                    guest.id,
+                    owner.id,
+                );
+            }
+        });
+
+        it('a share that cannot be checked', async () => {
+            const app = await makeApp();
+            const { handle } = await share(app);
+            const lookup = vi
+                .spyOn(env.server.stores.kvShareHandle, 'getLive')
+                .mockRejectedValue(new Error('lookup unavailable'));
+            try {
+                await expect(
+                    send(inApp(owner, app), { to: { handle } }),
+                ).rejects.toMatchObject({
+                    statusCode: 503,
+                    legacyCode: 'service_unavailable',
+                });
+            } finally {
+                lookup.mockRestore();
+            }
+        });
+    });
+});

src/backend/drivers/notification/NotificationDriver.sendAt.test.ts

@@ -0,0 +1,493 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * `notify({ sendAt })` without and with `services.schedules`, and the deferred
+ * mailbox write an extension runs inside its own transaction, fenced on a guard
+ * in a table of its own (a test table here).
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { runWithContext } from '../../core/context.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    DeferredSendPlan,
+    ScheduleServiceContract,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { EVENTS_BACKGROUND_PERMISSION } from '../../services/events/authorization.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import type {
+    DeferredWriteTx,
+    NotificationDriver,
+} from './NotificationDriver.js';
+
+vi.setConfig({ testTimeout: 120_000 });
+
+const BOOT_TIMEOUT_MS = 480_000;
+const TEST_TABLE = 'deferred_send_test';
+
+let server: PuterServer;
+let holder: Actor;
+let app: { id: number; uid: string };
+let saved: unknown;
+let nextId = 1;
+
+const db = () => server.clients.db;
+const driver = () =>
+    server.drivers.notifications as unknown as NotificationDriver;
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+const asApp = (): Actor => makeActor({ user: holder.user, app });
+
+const notify = (fields: Record<string, unknown>): Promise<unknown> =>
+    runWithContext({ actor: asApp() }, () =>
+        driver().create({
+            object: { value: { type: 'app.message', title: 'Hi', ...fields } },
+        }),
+    );
+
+const codeOf = async (run: Promise<unknown>): Promise<string | undefined> => {
+    try {
+        await run;
+    } catch (err) {
+        if (isHttpError(err)) return err.legacyCode;
+        throw err;
+    }
+    return undefined;
+};
+
+const mailbox = () =>
+    db().read(
+        'SELECT * FROM `notification` WHERE `user_id` = ? AND `app_uid` = ? ' +
+            'ORDER BY `id`',
+        [holder.user!.id, app.uid],
+    );
+
+/** A row's value: JSON text on sqlite, parsed on the other engines. */
+const valueOf = (row: Record<string, unknown>) =>
+    typeof row.value === 'string' ? JSON.parse(row.value) : row.value;
+
+/** A send claimed under `token`, as an extension's own row would be. */
+const claimed = async (token: string): Promise<number> => {
+    const id = nextId++;
+    await db().write(
+        `INSERT INTO \`${TEST_TABLE}\` (\`id\`, \`token\`, \`state\`) VALUES (?, ?, 'sending')`,
+        [id, token],
+    );
+    return id;
+};
+
+/** The transaction an extension wraps the write in: its row, then the mailbox. */
+const txFor = (id: number, token: string): DeferredWriteTx => ({
+    guard: {
+        table: TEST_TABLE,
+        where: "`id` = ? AND `token` = ? AND `state` = 'delivered'",
+        values: [id, token],
+    },
+    async commit(entries: readonly BatchEntry[]) {
+        await db().batchWrite([
+            {
+                statement:
+                    `UPDATE \`${TEST_TABLE}\` SET \`state\` = 'delivered' ` +
+                    "WHERE `id` = ? AND `token` = ? AND `state` = 'sending'",
+                values: [id, token],
+            },
+            ...entries,
+        ]);
+        const [row] = await db().pread(
+            `SELECT \`state\`, \`token\` FROM \`${TEST_TABLE}\` WHERE \`id\` = ?`,
+            [id],
+        );
+        return row?.state === 'delivered' && row.token === token;
+    },
+});
+
+const write = (
+    sendKey: string,
+    tx: DeferredWriteTx,
+    over: {
+        payload?: string | null;
+        appUid?: string;
+        signal?: AbortSignal;
+    } = {},
+) =>
+    driver().deferredAppMessages.write(
+        {
+            holderUserId: holder.user!.id,
+            appUid: over.appUid ?? app.uid,
+            payload:
+                over.payload === undefined
+                    ? JSON.stringify({ message: { title: 'Later' } })
+                    : over.payload,
+            sendKey,
+            signal: over.signal ?? new AbortController().signal,
+        },
+        tx,
+    );
+
+const fakeSchedules = (plan: Partial<DeferredSendPlan>) =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(),
+        list: vi.fn(),
+        get: vi.fn(),
+        remove: vi.fn(),
+        receivePeerChanges: vi.fn(),
+        prepareSendAt: vi.fn(async (): Promise<DeferredSendPlan> => ({
+            sendAt: 0,
+            immediate: false,
+            holderUserId: holder.user!.id,
+            appUid: app.uid,
+            handlerDepth: 0,
+            quota: {
+                recurringPerUser: 1,
+                recurringPerApp: 1,
+                oneShotsPerUser: 1,
+                oneShotsPerApp: 1,
+                budgetPerUser: 1,
+                budgetPerApp: 1,
+            },
+            ...plan,
+        })),
+        createDeferredAppMessage: vi.fn(async (p: DeferredSendPlan) => ({
+            scheduleId: 'sched-1',
+            sendAt: p.sendAt,
+        })),
+    }) satisfies ScheduleServiceContract;
+
+beforeAll(async () => {
+    server = await setupTestServer({
+        events: { enabled: true },
+    } as IConfig);
+    const username = `sendat-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+        requires_email_confirmation: false,
+    });
+    const row = (await server.stores.user.getById(user.id))!;
+    holder = makeActor({
+        user: {
+            id: row.id,
+            uuid: row.uuid,
+            username: row.username,
+            email: row.email ?? null,
+            email_confirmed: true,
+        } as Actor['user'],
+    });
+    const name = `sendat-app-${Math.random().toString(36).slice(2, 10)}`;
+    app = (await server.stores.app.create(
+        { name, title: name, index_url: `https://${name}.test/` },
+        { ownerUserId: row.id },
+    )) as { id: number; uid: string };
+    await server.services.permission.grantUserAppPermission(
+        holder,
+        app.uid,
+        EVENTS_BACKGROUND_PERMISSION,
+    );
+    await db().write(
+        `CREATE TABLE \`${TEST_TABLE}\` (\`id\` INTEGER PRIMARY KEY, ` +
+            '`token` VARCHAR(64) NOT NULL, `state` VARCHAR(16) NOT NULL)',
+        [],
+    );
+    saved = slot().schedules;
+}, BOOT_TIMEOUT_MS);
+
+beforeEach(async () => {
+    await db().write('DELETE FROM `notification` WHERE `app_uid` = ?', [
+        app.uid,
+    ]);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+    vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+describe('notify({ sendAt }) without services.schedules', () => {
+    it('is refused events_disabled, even when due now, storing nothing', async () => {
+        setSchedules(undefined);
+        expect(await codeOf(notify({ sendAt: Date.now() }))).toBe(
+            'events_disabled',
+        );
+        expect(await codeOf(notify({ sendAt: Date.now() + 3_600_000 }))).toBe(
+            'events_disabled',
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends as before without sendAt', async () => {
+        setSchedules(undefined);
+        const sent = (await notify({ text: 'now' })) as { uid: string };
+        expect(sent.uid).toBeTruthy();
+        expect(await mailbox()).toHaveLength(1);
+    });
+});
+
+describe('notify({ sendAt }) with services.schedules', () => {
+    it('hands a later send to createDeferredAppMessage and answers its id', async () => {
+        const at = Date.now() + 3_600_000;
+        const schedules = fakeSchedules({ sendAt: at });
+        setSchedules(schedules);
+
+        await expect(notify({ sendAt: at, text: 'later' })).resolves.toEqual({
+            scheduled: { scheduleId: 'sched-1', sendAt: at },
+        });
+        expect(schedules.prepareSendAt).toHaveBeenCalledWith(
+            expect.objectContaining({ user: holder.user }),
+            at,
+        );
+        expect(schedules.createDeferredAppMessage).toHaveBeenCalledWith(
+            expect.objectContaining({ sendAt: at }),
+            {
+                message: expect.objectContaining({
+                    title: 'Hi',
+                    text: 'later',
+                }),
+            },
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends at once when the plan says it is due', async () => {
+        const schedules = fakeSchedules({ immediate: true });
+        setSchedules(schedules);
+
+        const sent = (await notify({ sendAt: Date.now() })) as {
+            uid: string;
+        };
+        expect(sent.uid).toBeTruthy();
+        expect(schedules.createDeferredAppMessage).not.toHaveBeenCalled();
+        expect(await mailbox()).toHaveLength(1);
+    });
+
+    it('passes a refusal from prepareSendAt through', async () => {
+        const schedules = fakeSchedules({});
+        schedules.prepareSendAt.mockRejectedValueOnce(
+            Object.assign(new Error('scoped'), {}),
+        );
+        setSchedules(schedules);
+        await expect(notify({ sendAt: Date.now() })).rejects.toThrow('scoped');
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+describe('deferredAppMessages.write', () => {
+    it('writes one row keyed by the send, in the caller`s transaction', async () => {
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(write(key, txFor(id, token))).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: key, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
+    });
+
+    it('writes nothing when the guard no longer holds: another claim took it', async () => {
+        const id = await claimed(uuidv4());
+        const key = uuidv4();
+
+        await expect(write(key, txFor(id, uuidv4()))).resolves.toBe('lost');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('replaces the open row of its group, which takes the key', async () => {
+        const open = (await notify({ group: 'g', text: 'first' })) as {
+            uid: string;
+        };
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(
+            write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g' },
+                }),
+            }),
+        ).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: open.uid, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
+    });
+
+    it('drops a payload it can`t read, and a sender whose app is gone', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        await expect(
+            write(uuidv4(), txFor(id, token), { payload: '{"nope":1}' }),
+        ).resolves.toBe('dropped');
+        await expect(
+            write(uuidv4(), txFor(id, token), { appUid: `app-${uuidv4()}` }),
+        ).resolves.toBe('dropped');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('writes nothing once the caller`s claim is lost', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        const lost = new AbortController();
+        lost.abort();
+
+        await expect(
+            write(uuidv4(), txFor(id, token), { signal: lost.signal }),
+        ).rejects.toThrow();
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+/**
+ * A second claim taking a send over while the first one's transaction is open.
+ * Needs two writers at once with row locks; sqlite serialises writers, so this
+ * runs on postgres only.
+ */
+describe.runIf(process.env.PUTER_TEST_DB_ENGINE === 'postgres')(
+    'a deferred write racing another writer',
+    () => {
+        /**
+         * Hold the first statement matching `match` on its connection, before
+         * or after it runs, until released.
+         */
+        const holdStatement = (match: RegExp, when: 'before' | 'after') => {
+            const client = db() as unknown as {
+                query: (
+                    target: unknown,
+                    sql: string,
+                    params?: unknown[],
+                ) => Promise<unknown>;
+            };
+            const original = client.query.bind(client);
+            let reached!: () => void;
+            let release!: () => void;
+            const atBarrier = new Promise<void>((r) => (reached = r));
+            const released = new Promise<void>((r) => (release = r));
+            let held = false;
+            vi.spyOn(client as never, 'query' as never).mockImplementation(
+                (async (target: unknown, sql: string, params?: unknown[]) => {
+                    if (held || !match.test(sql))
+                        return original(target, sql, params);
+                    held = true;
+                    if (when === 'before') {
+                        reached();
+                        await released;
+                        return original(target, sql, params);
+                    }
+                    const result = await original(target, sql, params);
+                    reached();
+                    await released;
+                    return result;
+                }) as never,
+            );
+            return { atBarrier, release };
+        };
+
+        const DELIVERED = /SET `state` = 'delivered'/;
+        const REPLACE = /`replaced_at` = \?, `send_key` = \? WHERE/;
+
+        /** Whether `promise` is still pending after `ms`. */
+        const stillPending = async (promise: Promise<unknown>, ms: number) => {
+            let settled = false;
+            const watched = (async () => {
+                try {
+                    await promise;
+                } finally {
+                    settled = true;
+                }
+            })();
+            await new Promise((r) => setTimeout(r, ms));
+            void watched;
+            return !settled;
+        };
+
+        it('a transaction that opened before a takeover writes nothing', async () => {
+            const stale = uuidv4();
+            const id = await claimed(stale);
+            const key = uuidv4();
+            const gate = holdStatement(DELIVERED, 'before');
+            const first = write(key, txFor(id, stale));
+            await gate.atBarrier;
+
+            await db().write(
+                `UPDATE \`${TEST_TABLE}\` SET \`token\` = ? WHERE \`id\` = ?`,
+                [uuidv4(), id],
+            );
+            gate.release();
+            await expect(first).resolves.toBe('lost');
+            expect(
+                await server.stores.notification.getBySendKey(key),
+            ).toBeNull();
+            expect(await mailbox()).toEqual([]);
+        });
+
+        it('an immediate replacement waiting on the open transaction lands after it, without the key', async () => {
+            const open = (await notify({ group: 'g', text: 'first' })) as {
+                uid: string;
+            };
+            const token = uuidv4();
+            const id = await claimed(token);
+            const key = uuidv4();
+            const gate = holdStatement(REPLACE, 'after');
+            const deferred = write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g', text: 'later' },
+                }),
+            });
+            await gate.atBarrier;
+
+            const immediate = notify({ group: 'g', text: 'newer' });
+            // Past the group lock's wait, then blocked on the row.
+            expect(await stillPending(immediate, 2_500)).toBe(true);
+            gate.release();
+            await expect(deferred).resolves.toBe('delivered');
+            await immediate;
+
+            const rows = await mailbox();
+            expect(rows).toHaveLength(1);
+            expect(rows[0]).toMatchObject({ uid: open.uid, send_key: null });
+            expect(valueOf(rows[0]).text).toBe('newer');
+        });
+    },
+);

src/backend/drivers/notification/NotificationDriver.ts

@@ -37,6 +37,7 @@ import {
     APP_MESSAGE_TYPE,
     appMessageSender,
     parseAppMessage,
+    type AppMessage,
 } from '../../services/notification/appMessage.js';
 import {
     APP_READABLE_AUDIENCES,
@@ -50,6 +51,15 @@ import {
     resolveNotificationWrite,
     type NotificationTypeName,
 } from '../../services/notification/notificationTypes.js';
+import {
+    schedulesUnavailable,
+    type DeferredAppMessage,
+    type DeferredSendPlan,
+    type ScheduleServiceContract,
+    type SqlGuard,
+} from '../../core/schedules.js';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
+import { withRedisLock } from '../../util/redisLock.js';
 import { PuterDriver } from '../types.js';
 import type { Actor } from '../../core/actor.js';
 import type { DriverRateLimitConfig } from '../meta.js';
@@ -58,6 +68,50 @@ import { CRUD_CONCURRENT } from '../util/crudLimits.js';
 const MAX_SELECT_LIMIT = 200;
 const MINUTE_MS = 60_000;
 const DAY_MS = 24 * 60 * MINUTE_MS;
+// A group's sends queue for up to ~1 s; a stalled holder blocks them for the TTL.
+const GROUP_LOCK_MS = 5_000;
+const GROUP_LOCK_ATTEMPTS = 40;
+const GROUP_LOCK_RETRY_MS = 25;
+
+type AppMessageWindow = 'minute' | 'day';
+
+export type { SqlGuard };
+
+/** What a delayed app message's fire-time write needs from its row. */
+export interface DeferredWriteInput {
+    holderUserId: number;
+    appUid: string;
+    /** The stored {@link DeferredAppMessage}, as JSON text. */
+    payload: string | null;
+    /** The slot's key: the written row's uid or `send_key`, and its `sendId`. */
+    sendKey: string;
+    /** Aborted when the caller's claim is lost. */
+    signal: AbortSignal;
+}
+
+/**
+ * The caller's transaction the mailbox write commits in. Atomicity is the
+ * caller's to keep; the type can't enforce it.
+ */
+export interface DeferredWriteTx {
+    /**
+     * What every mailbox statement is fenced on: the caller's row marked
+     * delivered by this claim.
+     */
+    guard: SqlGuard;
+    /**
+     * Must be the whole delivery in one step: the caller's guarded transition
+     * of its row to delivered and `entries`, in that order, in one
+     * `batchWrite`, then a read of the row on the primary. True only when that
+     * read shows this claim holding the row as delivered. Splitting the
+     * transition from `entries` lets one slot write two rows.
+     */
+    commit(entries: readonly BatchEntry[]): Promise<boolean>;
+}
+
+/** How a delayed send's write stood when it stopped. */
+export type DeferredWriteOutcome =
+    'delivered' | 'dropped' | 'lost' | 'senderSuspended';
 
 /** The mailbox slice a call named, in the terms the query takes it. */
 interface MailboxScope {
@@ -66,12 +120,71 @@ interface MailboxScope {
     appUid: string | null | undefined;
 }
 
+const assertValueFits = (value: Record<string, unknown>): void => {
+    if (
+        Buffer.byteLength(JSON.stringify(value), 'utf8') >
+        NOTIFICATION_VALUE_MAX_BYTES
+    ) {
+        throw new HttpError(
+            400,
+            `\`value\` may not exceed ${NOTIFICATION_VALUE_MAX_BYTES} bytes`,
+            { legacyCode: 'notification_value_too_large' },
+        );
+    }
+};
+
 const typeNotAllowed = (message: string): HttpError =>
     new HttpError(403, message, {
         legacyCode: 'forbidden',
         code: 'notification_type_not_allowed',
     });
 
+const tooManyAppMessages = (): HttpError =>
+    new HttpError(429, 'Too many notifications from this app.', {
+        legacyCode: 'too_many_requests',
+    });
+
+/** One refusal for either limit a notification through a share can hit. */
+const tooManySharedNotifications = (): HttpError =>
+    new HttpError(429, 'Too many notifications.', {
+        legacyCode: 'too_many_requests',
+    });
+
+/** The handle in `to`, which must be `{ handle }` and nothing else. */
+const parseNotifyTarget = (to: unknown): string => {
+    const target = to as Record<string, unknown> | null;
+    if (
+        !target ||
+        typeof target !== 'object' ||
+        Array.isArray(target) ||
+        Object.keys(target).length !== 1 ||
+        typeof target.handle !== 'string' ||
+        target.handle === ''
+    ) {
+        throw new HttpError(
+            400,
+            '`to` must be an object with only a string `handle`',
+            { legacyCode: 'bad_request', fields: { field: 'to' } },
+        );
+    }
+    return target.handle;
+};
+
+/** A stored delayed message, or `null` for one that can't be read back. */
+const parseDelayedAppMessage = (
+    payload: string | null,
+): DeferredAppMessage | null => {
+    try {
+        const stored = JSON.parse(payload ?? 'null') as DeferredAppMessage;
+        if (typeof stored?.message?.title !== 'string') return null;
+        if (stored.to !== undefined && typeof stored.to !== 'string')
+            return null;
+        return stored;
+    } catch {
+        return null;
+    }
+};
+
 /** Whether a row belongs to the slice, as the same SQL scope selected it. */
 const inScope = (
     row: Record<string, unknown>,
@@ -102,8 +215,8 @@ const inScope = (
  *   grant changes that. A slice an actor may not see comes back empty rather
  *   than refused — the mailbox is not an oracle.
  * - The holder creates any registered type but `app.message`, silently. An actor
- *   holding an app creates `app.message` only, for its own user, and it is
- *   delivered live.
+ *   holding an app creates `app.message` only, for its own user or, through a
+ *   KV share handle, for the share's other party, and it is delivered live.
  *
  * `read` and `select` also take an optional `subject` naming one slice
  * (`notif:<appUid>:<audience>`); the two-segment form expands from the actor's
@@ -155,17 +268,7 @@ export class NotificationDriver extends PuterDriver {
                 legacyCode: 'bad_request',
             });
         }
-        if (
-            Buffer.byteLength(JSON.stringify(value), 'utf8') >
-            NOTIFICATION_VALUE_MAX_BYTES
-        ) {
-            throw new HttpError(
-                400,
-                `\`value\` may not exceed ${NOTIFICATION_VALUE_MAX_BYTES} bytes`,
-                { legacyCode: 'notification_value_too_large' },
-            );
-        }
-
+        // Its own field checks come first, so an app hears which field is wrong.
         if (actor.effectiveApp) {
             return this.#createAppMessage(
                 actor,
@@ -174,6 +277,7 @@ export class NotificationDriver extends PuterDriver {
                 value,
             );
         }
+        assertValueFits(value);
         if (type === APP_MESSAGE_TYPE) {
             throw typeNotAllowed(`Only apps can create ${APP_MESSAGE_TYPE}`);
         }
@@ -198,11 +302,12 @@ export class NotificationDriver extends PuterDriver {
         // Silent: a driver `create` has never pushed to the live socket, only
         // ever written the row — moving through the registry does not change
         // that.
-        return this.#write(actor, value, {
+        const uid = await this.#write(actor.user.id, value, {
             type: registeredType,
             appUid,
             silent: true,
         });
+        return this.#readBack(uid, actor.user.id);
     }
 
     async read(args: Record<string, unknown>): Promise<unknown> {
@@ -300,14 +405,17 @@ export class NotificationDriver extends PuterDriver {
         return { success: !!ok };
     }
 
-    /** An app's message to its own user; the actor names recipient and app. */
+    /**
+     * An app's message to its own user, or with `to.handle` to the other party
+     * of a KV share. The actor names the app; the share names the recipient.
+     */
     async #createAppMessage(
         actor: UserActor,
         appUid: string,
         object: Record<string, unknown>,
         value: Record<string, unknown>,
     ): Promise<unknown> {
-        const { type, ...fields } = value;
+        const { type, to, sendAt, ...fields } = value;
         if (type !== APP_MESSAGE_TYPE) {
             throw typeNotAllowed(`Apps can create only ${APP_MESSAGE_TYPE}`);
         }
@@ -320,70 +428,345 @@ export class NotificationDriver extends PuterDriver {
                 );
             }
         }
+        const handle = to === undefined ? undefined : parseNotifyTarget(to);
         const message = parseAppMessage(fields);
+        assertValueFits(value);
+        const timing =
+            sendAt === undefined || sendAt === null
+                ? null
+                : await this.#schedules().prepareSendAt(actor, sendAt);
+        if (timing && !timing.immediate)
+            return this.#scheduleAppMessage(actor, message, handle, timing);
 
-        if (!(await this.#withinAppMessageLimits(actor.user.id, appUid))) {
-            throw new HttpError(429, 'Too many notifications from this app.', {
-                legacyCode: 'too_many_requests',
-            });
+        if (handle === undefined) {
+            const uid = await this.#sendAppMessage(
+                actor.user.id,
+                appUid,
+                message,
+            );
+            return this.#readBack(uid, actor.user.id);
         }
+        try {
+            const recipient =
+                await this.services.kvShare.resolveNotifyRecipient(
+                    actor,
+                    handle,
+                );
+            await this.#sendAppMessage(
+                recipient.recipientUserId,
+                recipient.appUid,
+                message,
+            );
+        } catch (err) {
+            // Which limit refused would say how much the recipient is sent.
+            if (err instanceof HttpError && err.statusCode === 429)
+                throw tooManySharedNotifications();
+            throw err;
+        }
+        // Even a uid would say whether the recipient's last one was replaced.
+        return {};
+    }
+
+    /**
+     * A message with a `group` replaces the newest one in that group unless it
+     * was shown or dismissed, which spends a minute slot but no day slot.
+     * Limits are keyed on the recipient, who need not be the actor.
+     */
+    async #sendAppMessage(
+        recipientId: number,
+        appUid: string,
+        message: AppMessage,
+    ): Promise<string> {
+        return this.#oneSendPerGroup(recipientId, appUid, message, () =>
+            this.#replaceOrWriteAppMessage(recipientId, appUid, message),
+        );
+    }
+
+    /**
+     * One send per group at a time, so a burst into an empty group writes one
+     * row that the rest replace. A lock that is busy past the wait or
+     * unreachable still sends: the race costs a second row and its day slot.
+     */
+    #oneSendPerGroup<T>(
+        recipientId: number,
+        appUid: string,
+        message: AppMessage,
+        send: () => Promise<T>,
+    ): Promise<T> {
+        if (!message.group) return send();
+        return withRedisLock(
+            this.clients.redis,
+            `notif:app-message:group:${recipientId}:${appUid}:${message.group}`,
+            send,
+            {
+                ttlMs: GROUP_LOCK_MS,
+                attempts: GROUP_LOCK_ATTEMPTS,
+                retryMs: GROUP_LOCK_RETRY_MS,
+                onUnavailable: 'run',
+                onBusy: send,
+            },
+        );
+    }
+
+    // -- Delayed sends -----------------------------------------------
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /**
+     * Store an app's message to go out at `sendAt`. A recipient named through a
+     * share is checked now without spending its budget; every limit is spent
+     * when it goes, and the share is checked again then.
+     */
+    async #scheduleAppMessage(
+        actor: UserActor,
+        message: AppMessage,
+        handle: string | undefined,
+        plan: DeferredSendPlan,
+    ): Promise<{ scheduled: { scheduleId: string; sendAt: number } }> {
+        if (handle !== undefined)
+            await this.services.kvShare.resolveNotifyRecipient(actor, handle, {
+                spendBudget: false,
+            });
+        return {
+            scheduled: await this.#schedules().createDeferredAppMessage(
+                plan,
+                handle === undefined ? { message } : { message, to: handle },
+            ),
+        };
+    }
+
+    /**
+     * For extensions. A field, not a method, so `/drivers/call` can't reach it.
+     */
+    readonly deferredAppMessages = {
+        write: (input: DeferredWriteInput, tx: DeferredWriteTx) =>
+            this.#writeDeferredAppMessage(input, tx),
+    };
+
+    /**
+     * Check the sender and recipient as they stand now, spend the recipient's
+     * limits, and write the mailbox row through `tx`, fenced on its guard.
+     * Refusals are the same `HttpError`s an immediate send throws.
+     */
+    async #writeDeferredAppMessage(
+        input: DeferredWriteInput,
+        tx: DeferredWriteTx,
+    ): Promise<DeferredWriteOutcome> {
+        const { holderUserId, sendKey, signal } = input;
+        const sender = await this.services.events.resolveScheduleActor(
+            holderUserId,
+            input.appUid,
+        );
+        if (!sender) {
+            // A lagging replica can miss an app the primary has: retried.
+            const there =
+                (await this.stores.app.uidExistsOnPrimary(input.appUid)) &&
+                (await this.stores.user.getById(holderUserId)) !== null;
+            if (there) throw new Error('the sender could not be resolved');
+            return 'dropped';
+        }
+        if (sender.user?.suspended) return 'senderSuspended';
+        // Uninstalled since, and the cancel at uninstall missed it.
+        const appId = sender.effectiveApp?.id;
+        if (
+            typeof appId !== 'number' ||
+            !(await this.stores.permission.hasAnyUserAppPerm(
+                holderUserId,
+                appId,
+            ))
+        )
+            return 'dropped';
+        const stored = parseDelayedAppMessage(input.payload);
+        if (!stored) return 'dropped';
+        const { message } = stored;
+
+        let recipientId = holderUserId;
+        let appUid = input.appUid;
+        if (stored.to !== undefined) {
+            const recipient =
+                await this.services.kvShare.resolveNotifyRecipient(
+                    sender,
+                    stored.to,
+                );
+            recipientId = recipient.recipientUserId;
+            appUid = recipient.appUid;
+        }
+
+        return this.#oneSendPerGroup(recipientId, appUid, message, async () => {
+            // Primary, as for an immediate send.
+            const open = message.group
+                ? await this.stores.notification.getUnseenByGroup(
+                      recipientId,
+                      appUid,
+                      message.group,
+                      { primary: true },
+                  )
+                : null;
+            await this.#spendAppMessageLimits(
+                recipientId,
+                appUid,
+                open ? ['minute'] : ['minute', 'day'],
+            );
+            message.app = appMessageSender(
+                appUid,
+                await this.stores.app.getByUid(appUid),
+            );
+            // Another attempt may hold the lease now; hand the claim back.
+            if (signal.aborted) throw new Error('delayed send lost its lease');
+
+            const registered = resolveNotificationWrite(
+                APP_MESSAGE_TYPE,
+                appUid,
+            );
+            const delivered = await tx.commit(
+                this.stores.notification.sendEntries(
+                    {
+                        sendKey,
+                        userId: recipientId,
+                        value: { ...message, type: registered.type },
+                        type: registered.type,
+                        audience: registered.audience,
+                        appUid,
+                        group: message.group ?? null,
+                        replaceUid: open ? String(open.uid) : null,
+                        at: Date.now(),
+                    },
+                    tx.guard,
+                ),
+            );
+            if (!delivered) return 'lost';
+            if (open)
+                await this.#spendDayIfFresh(recipientId, appUid, open, sendKey);
+            return 'delivered';
+        });
+    }
+
+    /**
+     * A replacement that found its row shown since the lookup wrote a new one,
+     * which costs a day slot. Already written, so a full day is not a refusal.
+     */
+    async #spendDayIfFresh(
+        recipientId: number,
+        appUid: string,
+        open: Record<string, unknown>,
+        sendKey: string,
+    ): Promise<void> {
+        try {
+            const written =
+                await this.stores.notification.getBySendKey(sendKey);
+            if (written && written.uid !== open.uid)
+                await this.#spendAppMessageLimits(recipientId, appUid, ['day']);
+        } catch {
+            // The row is written; a slot not spent is the only loss.
+        }
+    }
+
+    async #replaceOrWriteAppMessage(
+        recipientId: number,
+        appUid: string,
+        message: AppMessage,
+    ): Promise<string> {
+        // Primary: a lagging replica would turn a replacement into a new
+        // notification, which spends a day slot.
+        const open = message.group
+            ? await this.stores.notification.getUnseenByGroup(
+                  recipientId,
+                  appUid,
+                  message.group,
+                  { primary: true },
+              )
+            : null;
+        await this.#spendAppMessageLimits(
+            recipientId,
+            appUid,
+            open ? ['minute'] : ['minute', 'day'],
+        );
 
         message.app = appMessageSender(
             appUid,
             await this.stores.app.getByUid(appUid),
         );
 
-        return this.#write(actor, message, {
+        if (open) {
+            const uid = String(open.uid);
+            if (
+                await this.services.notification.notifyUpdate(
+                    uid,
+                    recipientId,
+                    message,
+                    { type: APP_MESSAGE_TYPE, replace: true },
+                )
+            ) {
+                return uid;
+            }
+            // Shown or dismissed since the lookup, so this one is new after all.
+            await this.#spendAppMessageLimits(recipientId, appUid, ['day']);
+        }
+
+        return this.#write(recipientId, message, {
             type: APP_MESSAGE_TYPE,
             appUid,
             silent: false,
+            group: message.group ?? null,
         });
     }
 
     /**
-     * Both windows are read before either is spent, so a refusal costs none.
+     * Every window named is read before any is spent, so a refusal costs none.
      * The minute window is spent first: it is the one most likely to run out
      * between the read and the spend, and a refusal there leaves the day
      * alone.
      */
-    async #withinAppMessageLimits(
+    async #spendAppMessageLimits(
         userId: number,
         appUid: string,
-    ): Promise<boolean> {
+        spend: readonly AppMessageWindow[],
+    ): Promise<void> {
         const configured = this.config.notificationAppMessageLimits ?? {};
-        const all: Array<[string, number, number]> = [
-            [
+        const all: Record<AppMessageWindow, [string, number, number]> = {
+            minute: [
                 `notif:app-message:minute:${userId}:${appUid}`,
                 configured.perMinute ?? APP_MESSAGE_RATE_LIMITS.perMinute,
                 MINUTE_MS,
             ],
-            [
+            day: [
                 `notif:app-message:day:${userId}:${appUid}`,
                 configured.perDay ?? APP_MESSAGE_RATE_LIMITS.perDay,
                 DAY_MS,
             ],
-        ];
-        const windows = all.filter(([, limit]) => limit > 0);
+        };
+        const windows = spend
+            .map((name) => all[name])
+            .filter(([, limit]) => limit > 0);
         for (const [key, limit, windowMs] of windows) {
-            if (!(await peekRateLimit(key, limit, windowMs))) return false;
+            if (!(await peekRateLimit(key, limit, windowMs))) {
+                throw tooManyAppMessages();
+            }
         }
         for (const [key, limit, windowMs] of windows) {
-            if (!(await checkRateLimit(key, limit, windowMs))) return false;
+            if (!(await checkRateLimit(key, limit, windowMs))) {
+                throw tooManyAppMessages();
+            }
         }
-        return true;
     }
 
+    /** Writes for `userId`, who need not be the actor, and names the row. */
     async #write(
-        actor: UserActor,
+        userId: number,
         value: Record<string, unknown>,
         opts: {
             type: NotificationTypeName;
             appUid: string | null;
             silent: boolean;
+            group?: string | null;
         },
-    ): Promise<unknown> {
+    ): Promise<string> {
         const uid = await this.services.notification.notify(
-            [actor.user.id],
+            [userId],
             value,
             opts,
         );
@@ -392,10 +775,17 @@ export class NotificationDriver extends PuterDriver {
                 legacyCode: 'internal_error',
             });
         }
-        // `notify` hands back only the uid; the primary has the rest.
+        return uid;
+    }
+
+    /** The service hands back only the uid; the primary has the rest. */
+    async #readBack(
+        uid: string,
+        userId: number,
+    ): Promise<Record<string, unknown> | null> {
         return this.#toClient(
             await this.stores.notification.getByUid(uid, {
-                userId: actor.user.id,
+                userId,
                 primary: true,
             }),
         );

src/backend/drivers/util/tokenEstimate.ts

@@ -17,6 +17,49 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+/** The code units `\s` matches. */
+const isSpace = (c: number): boolean =>
+    c === 0x20 ||
+    (c >= 0x09 && c <= 0x0d) ||
+    (c >= 0xa0 &&
+        (c === 0xa0 ||
+            c === 0x1680 ||
+            (c >= 0x2000 && c <= 0x200a) ||
+            c === 0x2028 ||
+            c === 0x2029 ||
+            c === 0x202f ||
+            c === 0x205f ||
+            c === 0x3000 ||
+            c === 0xfeff));
+
+/** Whitespace runs in `text.slice(0, end)`, one less than `split(/\s+/)` gives. */
+const countSpaceRuns = (text: string, end: number): number => {
+    let runs = 0;
+    let inSpace = false;
+    for (let i = 0; i < end; i++) {
+        const space = isSpace(text.charCodeAt(i));
+        if (space && !inSpace) runs++;
+        inSpace = space;
+    }
+    return runs;
+};
+
+/** How much text is left to scan for word boundaries, and what was seen. */
+export interface WordScanBudget {
+    chars: number;
+    scannedChars: number;
+    scannedRuns: number;
+}
+
+/** About 5 ms of scanning. */
+const WORD_SCAN_CHARS = 1024 * 1024;
+
+export const createWordScanBudget = (): WordScanBudget => ({
+    chars: WORD_SCAN_CHARS,
+    scannedChars: 0,
+    scannedRuns: 0,
+});
+
 /**
  * Tokens a run of text is worth.
  *
@@ -28,11 +71,27 @@
  * Shared by the chat credit gate, the chat unreported-stream backstop, and
  * image-prompt pricing, so a calibration moves all of them at once.
  *
+ * Words are counted over at most `budget.chars` characters, which this call
+ * spends; text past that is given the word density scanned so far. Pass one
+ * budget across a request's parts to bound the scan per request.
+ *
  * @see https://help.openai.com/en/articles/4936856
  */
-export const estimateTextTokens = (text: string): number => {
+export const estimateTextTokens = (
+    text: string,
+    budget: WordScanBudget = createWordScanBudget(),
+): number => {
     if (!text) return 0;
-    return Math.floor(
-        (text.length / 4 + text.split(/\s+/).length * (4 / 3)) / 2,
-    );
+    const scanned = Math.min(text.length, budget.chars);
+    const runs = countSpaceRuns(text, scanned);
+    budget.chars -= scanned;
+    budget.scannedChars += scanned;
+    budget.scannedRuns += runs;
+    const unscannedRuns =
+        budget.scannedChars > 0
+            ? ((text.length - scanned) * budget.scannedRuns) /
+              budget.scannedChars
+            : 0;
+    const words = 1 + runs + unscannedRuns;
+    return Math.floor((text.length / 4 + words * (4 / 3)) / 2);
 };

src/backend/drivers/workers/WorkerDriver.cloudflare.test.ts

@@ -238,6 +238,8 @@ describe('WorkerDriver.create with a configured deploy backend', () => {
             success: true,
             errors: [],
             url: `https://${name}.puter.work`,
+            app_uid: null,
+            app_name: null,
         });
 
         const [url, init] = putCalls()[0]!;
@@ -255,6 +257,165 @@ describe('WorkerDriver.create with a configured deploy backend', () => {
         expect(Number(row!.root_dir_id)).toBe(entry.id);
     });
 
+    it('names the app a bound worker runs as, and lists it with the worker', async () => {
+        const { user, actor } = await makeUser();
+        const app = await server.stores.app.create(
+            {
+                name: `bound-${user.username}`,
+                title: `bound-${user.username}`,
+                index_url: `https://bound-${user.username}.example.com/`,
+            },
+            { ownerUserId: user.id },
+        );
+        const path = `/${user.username}/bound.js`;
+        await writeSource(actor, user.id, path, 'export default {}');
+        const name = `bound-${user.username}`;
+
+        const result = await inCtx(actor, () =>
+            target.create({ appId: app.uid, workerName: name, filePath: path }),
+        );
+
+        expect(result).toMatchObject({
+            success: true,
+            app_uid: app.uid,
+            app_name: app.name,
+        });
+        const listed = (
+            (await inCtx(actor, () =>
+                target.getFilePaths({ workerName: name }),
+            )) as Array<{ app_uid: string | null; app_name: string | null }>
+        )[0];
+        expect(listed).toMatchObject({ app_uid: app.uid, app_name: app.name });
+    });
+
+    describe('redeployed under a different app', () => {
+        const seedBound = async () => {
+            const { user, actor } = await makeUser();
+            const makeApp = (label: string) =>
+                server.stores.app.create(
+                    {
+                        name: `${label}-${user.username}`,
+                        title: `${label}-${user.username}`,
+                        index_url: `https://${label}-${user.username}.example.com/`,
+                    },
+                    { ownerUserId: user.id },
+                );
+            const first = await makeApp('first');
+            const second = await makeApp('second');
+            const path = `/${user.username}/rebind.js`;
+            await writeSource(actor, user.id, path, 'v1');
+            const name = `rebind-${user.username}`;
+            const deploy = (appId: string) =>
+                inCtx(actor, () =>
+                    target.create({ appId, workerName: name, filePath: path }),
+                );
+            await deploy(first.uid);
+            const firstToken = deployedToken();
+            return {
+                user,
+                actor,
+                path,
+                name,
+                first,
+                second,
+                deploy,
+                firstToken,
+            };
+        };
+
+        const appOf = async (token: string) =>
+            (await server.services.auth.authenticate(token)).actor
+                ?.effectiveApp?.uid;
+
+        const listed = async (actor: Actor, name: string) =>
+            (
+                (await inCtx(actor, () =>
+                    target.getFilePaths({ workerName: name }),
+                )) as Array<{ app_uid: string | null; app_name: string | null }>
+            )[0];
+
+        it('moves the worker row to the app its new token acts as', async () => {
+            const { actor, name, second, deploy } = await seedBound();
+
+            const result = await deploy(second.uid);
+
+            expect(result).toMatchObject({ app_uid: second.uid });
+            expect(await listed(actor, name)).toMatchObject({
+                app_uid: second.uid,
+                app_name: second.name,
+            });
+            const row = await server.stores.subdomain.getBySubdomain(
+                `workers.puter.${name}`,
+            );
+            expect(Number(row!.app_owner)).toBe(second.id);
+        });
+
+        it('hot-reloads as the new app, not the one it was first bound to', async () => {
+            const { user, actor, path, second, deploy } = await seedBound();
+            await deploy(second.uid);
+            const notifySpy = vi.spyOn(server.services.notification, 'notify');
+
+            await writeSource(actor, user.id, path, 'v2');
+
+            await waitFor(
+                () => notifySpy.mock.calls.length > 0,
+                'hot-reload notification',
+            );
+            expect(notifySpy).toHaveBeenCalledWith(
+                [user.id],
+                expect.anything(),
+                expect.objectContaining({ appUid: second.uid }),
+            );
+        });
+
+        it('drops the binding when redeployed with no app', async () => {
+            const { actor, name, deploy, firstToken } = await seedBound();
+
+            expect(await deploy('')).toMatchObject({ app_uid: null });
+            expect(await listed(actor, name)).toMatchObject({
+                app_uid: null,
+                app_name: null,
+            });
+            expect(await server.services.auth.authenticate(firstToken)).toEqual(
+                { invalid: true },
+            );
+            expect(
+                (await server.services.auth.authenticate(deployedToken()))
+                    .actor,
+            ).toBeTruthy();
+        });
+
+        it('revokes the old app`s token once the new one is deployed', async () => {
+            const { second, deploy, firstToken } = await seedBound();
+
+            await deploy(second.uid);
+
+            expect(await server.services.auth.authenticate(firstToken)).toEqual(
+                { invalid: true },
+            );
+            expect(await appOf(deployedToken())).toBe(second.uid);
+        });
+
+        it('leaves the old binding and its token alone when the redeploy fails', async () => {
+            const { name, first, second, deploy, firstToken } =
+                await seedBound();
+            fetchSpy.mockResolvedValue(
+                edgeResponse({
+                    success: false,
+                    errors: [{ message: 'script too large' }],
+                }),
+            );
+
+            expect(await deploy(second.uid)).toMatchObject({ success: false });
+
+            expect(await appOf(firstToken)).toBe(first.uid);
+            const row = await server.stores.subdomain.getBySubdomain(
+                `workers.puter.${name}`,
+            );
+            expect(Number(row!.app_owner)).toBe(first.id);
+        });
+    });
+
     it('sends the puter_auth secret and puter_endpoint binding in the deploy metadata', async () => {
         const { user, actor } = await makeUser();
         const path = `/${user.username}/worker.js`;
@@ -1319,6 +1480,7 @@ describe('WorkerDriver.getFilePaths source resolution', () => {
             file_path: string | null;
             file_uid: string | null;
             app_uid: string | null;
+            app_name: string | null;
             created_at: string | null;
         }>;
 
@@ -1328,6 +1490,7 @@ describe('WorkerDriver.getFilePaths source resolution', () => {
         expect(row.file_path).toBe(path);
         expect(row.file_uid).toBe(entry.uuid);
         expect(row.app_uid).toBeNull();
+        expect(row.app_name).toBeNull();
         expect(row.created_at).toMatch(/^\d{4}-\d{2}-\d{2}T/);
     });
 

src/backend/drivers/workers/WorkerDriver.test.ts

@@ -509,7 +509,7 @@ describe('WorkerDriver', () => {
             );
         });
 
-        it('returns app_uid for sandboxed/app-bound workers and null for user-scoped workers', async () => {
+        it('returns app_uid and app_name for sandboxed/app-bound workers and null for user-scoped workers', async () => {
             const tag = `wkuid${Math.random().toString(36).slice(2, 8)}`;
             const owner = await makeUser(tag);
             const app = await server.stores.app.create(
@@ -538,15 +538,21 @@ describe('WorkerDriver', () => {
             const seen = (await inCtx(
                 () => target.getFilePaths({}),
                 actorFor(owner),
-            )) as Array<{ name: string; app_uid: string | null }>;
+            )) as Array<{
+                name: string;
+                app_uid: string | null;
+                app_name: string | null;
+            }>;
 
             const bound = seen.find((r) => r.name === `${tag}-bound`);
             const plain = seen.find((r) => r.name === `${tag}-plain`);
 
             expect(bound).toBeDefined();
             expect(bound?.app_uid).toBe(app.uid);
+            expect(bound?.app_name).toBe(`app-${tag}`);
             expect(plain).toBeDefined();
             expect(plain?.app_uid).toBeNull();
+            expect(plain?.app_name).toBeNull();
         });
     });
 
@@ -613,6 +619,7 @@ describe('WorkerDriver', () => {
                         file_path: dir.path,
                         file_uid: dir.uuid,
                         app_uid: app.uid,
+                        app_name: app.name,
                         created_at: expect.any(String),
                     },
                 ]);

src/backend/drivers/workers/WorkerDriver.ts

@@ -635,7 +635,18 @@ export class WorkerDriver extends PuterDriver {
             runtimePreamble.source + sourceCode,
             deployTarget,
         );
-        return cfResult;
+        if (existingSub && cfResult.success === true)
+            await this.#rebindAfterDeploy(existingSub, actor, workerName, {
+                appOwner: appOwnerId ?? null,
+                appUid: boundApp?.uid ?? null,
+            });
+        // The app the worker's token acts as: its KV, AppData, notifications
+        // and pushes are that app's.
+        return {
+            ...cfResult,
+            app_uid: boundApp?.uid ?? null,
+            app_name: boundApp?.name ?? null,
+        };
     }
 
     /** Tear down the caller's worker; a refused edge delete is a 502. */
@@ -700,6 +711,35 @@ export class WorkerDriver extends PuterDriver {
         };
     }
 
+    /**
+     * A redeploy under a different app (or none) moves the row to the app the
+     * new token acts as, so the listing, hot-reload and `create()` agree, and
+     * revokes the session the old binding ran on. Only once the new script is
+     * live: a failed redeploy leaves the old one running on its old token.
+     */
+    async #rebindAfterDeploy(
+        row: SubdomainRow,
+        actor: Actor & { user: { id: number } },
+        workerName: string,
+        next: { appOwner: number | null; appUid: string | null },
+    ): Promise<void> {
+        const current =
+            row.app_owner === null || row.app_owner === undefined
+                ? null
+                : Number(row.app_owner);
+        if (current === next.appOwner) return;
+        await this.stores.subdomain.setAppOwner(
+            String(row.uuid),
+            next.appOwner,
+            { userId: actor.user.id },
+        );
+        await this.services.auth.deleteWorkerSessionsByName(
+            workerName,
+            actor.user.id,
+            { exceptAppUid: next.appUid },
+        );
+    }
+
     /** Script, sessions, then row: a failed step leaves the row for a retry. */
     async #teardown(
         row: SubdomainRow,
@@ -827,17 +867,17 @@ export class WorkerDriver extends PuterDriver {
                 file_path = loaded?.path;
                 file_uid = loaded?.uuid;
             }
-            let app_uid = null;
-            if (typeof r.app_owner === 'number') {
-                const loadedApp = appsById.get(r.app_owner);
-                app_uid = loadedApp?.uid ?? null;
-            }
+            const loadedApp =
+                typeof r.app_owner === 'number'
+                    ? appsById.get(r.app_owner)
+                    : undefined;
             return {
                 name,
                 url: `https://${name}.puter.work`,
                 file_path,
                 file_uid,
-                app_uid,
+                app_uid: loadedApp?.uid ?? null,
+                app_name: loadedApp?.name ?? null,
                 created_at: r.ts
                     ? new Date(r.ts as string).toISOString()
                     : null,
@@ -1147,15 +1187,18 @@ export class WorkerDriver extends PuterDriver {
     async #resolveWorkerAppBinding(
         actor: Actor & { user: { id: number } },
         requestedAppUid?: string,
-    ): Promise<{ uid: string; id?: number } | null> {
+    ): Promise<{ uid: string; id?: number; name: string | null } | null> {
         // Self-binding, either implicit or named. Not every actor shape carries
-        // `app.id`, so fall back to a lookup rather than leaving the subdomain
-        // row unowned.
+        // `app.id`, and none carries the name, so both come from the row.
         const ownApp = actor.effectiveApp;
-        const selfBinding = async () => ({
-            uid: ownApp!.uid,
-            id: ownApp!.id ?? (await this.stores.app.getByUid(ownApp!.uid))?.id,
-        });
+        const selfBinding = async () => {
+            const row = await this.stores.app.getByUid(ownApp!.uid);
+            return {
+                uid: ownApp!.uid,
+                id: ownApp!.id ?? row?.id,
+                name: row?.name ?? null,
+            };
+        };
         if (!requestedAppUid) {
             return ownApp ? await selfBinding() : null;
         }
@@ -1168,7 +1211,11 @@ export class WorkerDriver extends PuterDriver {
             // Root session: unchanged: bind to whatever it names. An unknown
             // uid still mints a token, as it did before, and simply leaves the
             // subdomain row unowned by any app.
-            return { uid: requestedAppUid, id: app?.id };
+            return {
+                uid: requestedAppUid,
+                id: app?.id,
+                name: app?.name ?? null,
+            };
         }
 
         if (!app || !(await this.#appIsOwnedByActorApp(app.id, actor))) {
@@ -1176,7 +1223,7 @@ export class WorkerDriver extends PuterDriver {
                 legacyCode: 'forbidden',
             });
         }
-        return { uid: app.uid, id: app.id };
+        return { uid: app.uid, id: app.id, name: app.name ?? null };
     }
 
     /**

src/backend/package.json

@@ -53,11 +53,11 @@
     "mime-types": "^2.1.35",
     "murmurhash": "^2.0.1",
     "mysql2": "^3.24.6",
-    "nodemailer": "^9.1.1",
+    "nodemailer": "^10.1.4",
     "openai": "^6.34.0",
     "otpauth": "^9.5.2",
     "pg": "^8.21.0",
-    "postal-mime": "^3.0.1",
+    "postal-mime": "^4.2.0",
     "replicate": "^1.0.0",
     "sharp": "^0.35.5",
     "smtp-server": "^3.19.24",
@@ -67,7 +67,8 @@
     "ua-parser-js": "^1.0.41",
     "undici": "^7.30.1",
     "uuid": "^14.0.3",
-    "validator": "^13.15.35"
+    "validator": "^13.15.35",
+    "ws": "^8.22.0"
   },
   "devDependencies": {
     "@types/bcrypt": "^6.0.0",
@@ -81,6 +82,7 @@
     "@types/pg": "^8.6.1",
     "@types/smtp-server": "^3.5.13",
     "@types/validator": "^13.15.10",
+    "@types/ws": "^8.18.2",
     "pgmock": "^1.0.3",
     "typescript": "^5.9.3",
     "vite": "^8.1.6",

src/backend/server.test.ts

@@ -36,7 +36,15 @@ import { Controller, Get } from './core/http/decorators.ts';
 import { HttpError } from './core/http/HttpError.ts';
 import { extension, extensionStore } from './extensions.ts';
 import { PuterServer } from './server.ts';
-import { allocateEphemeralPort, setupTestServer } from './testUtil.ts';
+import {
+    JobScheduler,
+    type JobDefinition,
+} from './services/jobs/JobScheduler.ts';
+import {
+    allocateEphemeralPort,
+    createTestUser,
+    setupTestServer,
+} from './testUtil.ts';
 import type { IConfig } from './types';
 
 /**
@@ -523,6 +531,7 @@ describe('PuterServer host header validation — custom domains with every host
 describe('PuterServer request parsing', () => {
     let server: PuterServer;
     let port: number;
+    let token: string;
     const routeCount = { before: 0 };
 
     beforeAll(async () => {
@@ -551,6 +560,10 @@ describe('PuterServer request parsing', () => {
             } as unknown as IConfig,
             { listen: true },
         );
+        ({ token } = await createTestUser(server, {
+            username: 'large_body_user',
+            password: 'large-body-password',
+        }));
     });
 
     afterAll(async () => {
@@ -560,6 +573,85 @@ describe('PuterServer request parsing', () => {
 
     const host = () => `puter.localhost:${port}`;
 
+    const MiB = 1024 * 1024;
+    const postJson = (
+        path: string,
+        payload: string,
+        headers: Record<string, string> = {},
+    ) =>
+        rawRequest(
+            port,
+            path,
+            {
+                host: `api.puter.localhost:${port}`,
+                'content-type': 'application/json',
+                'content-length': String(Buffer.byteLength(payload)),
+                ...headers,
+            },
+            'POST',
+            payload,
+        );
+
+    it.each([
+        '/drivers/call',
+        '/puterai/openai/v1/chat/completions',
+        '/puterai/anthropic/v1/messages',
+        '/puterai/anthropic/v1/messages/count_tokens',
+    ])(
+        'keeps the default JSON limit on %s without a credential',
+        async (path) => {
+            const payload = JSON.stringify({ text: 'x'.repeat(51 * MiB) });
+            expect((await postJson(path, payload)).status).toBe(413);
+        },
+    );
+
+    it('parses a chat call above 50 MiB for a signed-in caller', async () => {
+        const payload = JSON.stringify({
+            interface: 'puter-chat-completion',
+            method: 'no-such-method',
+            pad: 'x'.repeat(51 * MiB),
+        });
+        for (const contentType of [
+            'application/json',
+            'text/plain;actually=json',
+        ]) {
+            const res = await postJson('/drivers/call', payload, {
+                'content-type': contentType,
+                authorization: `Bearer ${token}`,
+            });
+            // The controller read `method` out of the body.
+            expect(res.status, contentType).toBe(404);
+            expect(res.body).toContain('no-such-method');
+        }
+    });
+
+    it('keeps the default JSON limit for other drivers', async () => {
+        const payload = JSON.stringify({
+            interface: 'puter-kvstore',
+            method: 'get',
+            pad: 'x'.repeat(51 * MiB),
+        });
+        // Routing matches these spellings too.
+        for (const path of ['/drivers/call', '/Drivers/Call/']) {
+            const res = await postJson(path, payload, {
+                authorization: `Bearer ${token}`,
+            });
+            expect(res.status, path).toBe(413);
+        }
+    });
+
+    it.each([
+        ['/drivers/call', 128],
+        ['/puterai/anthropic/v1/messages', 128],
+        ['/raw-probe', 50],
+    ])('rejects bodies above the JSON limit on %s', async (path, limitMiB) => {
+        const payload = JSON.stringify({ text: 'x'.repeat(limitMiB * MiB) });
+        const res = await postJson(path, payload, {
+            authorization: `Bearer ${token}`,
+        });
+        expect(res.status).toBe(413);
+    });
+
     it('parses the user agent the first time a handler reads it', async () => {
         const agent =
             'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36';
@@ -1400,7 +1492,7 @@ describe('PuterServer keep-alive timeout', () => {
 });
 
 describe('PuterServer lifecycle hooks', () => {
-    it('runs prepare-shutdown bottom-up, once, before tearing down', async () => {
+    it('runs drain-start and prepare-shutdown bottom-up, once, before tearing down', async () => {
         const port = await allocateEphemeralPort();
         const server = await setupTestServer({ port } as unknown as IConfig, {
             listen: true,
@@ -1408,6 +1500,9 @@ describe('PuterServer lifecycle hooks', () => {
 
         const order: string[] = [];
         const probe = (label: string) => ({
+            onServerDrainStart: (info?: { drainMs?: number }) => {
+                order.push(`drain:${label}:${info?.drainMs}`);
+            },
             onServerPrepareShutdown: () => {
                 order.push(`prepare:${label}`);
             },
@@ -1425,10 +1520,15 @@ describe('PuterServer lifecycle hooks', () => {
             (layer as Record<string, unknown>).hookOrderProbe = probe(label);
         }
 
-        await server.prepareShutdown();
+        await server.prepareShutdown({ drainMs: 1_000 });
         await server.shutdown();
 
         expect(order).toEqual([
+            'drain:client:1000',
+            'drain:store:1000',
+            'drain:service:1000',
+            'drain:controller:1000',
+            'drain:driver:1000',
             'prepare:client',
             'prepare:store',
             'prepare:service',
@@ -1443,6 +1543,69 @@ describe('PuterServer lifecycle hooks', () => {
     });
 });
 
+describe('PuterServer shutdown during prepare', () => {
+    it('waits for prepare hooks the drain already started before tearing down', async () => {
+        const port = await allocateEphemeralPort();
+        const server = await setupTestServer({ port } as unknown as IConfig, {
+            listen: true,
+        });
+        const order: string[] = [];
+        (server.services as unknown as Record<string, unknown>).slowPrepare = {
+            onServerPrepareShutdown: async () => {
+                order.push('prepare:start');
+                await new Promise((resolve) => setTimeout(resolve, 200));
+                order.push('prepare:done');
+            },
+            onServerShutdown: () => {
+                order.push('shutdown');
+            },
+        };
+
+        await server.prepareShutdown({ drainMs: 0 });
+        // The listener's close callback starts the prepare hooks.
+        await new Promise((resolve) => setTimeout(resolve, 50));
+        await server.shutdown();
+
+        expect(order).toEqual(['prepare:start', 'prepare:done', 'shutdown']);
+    });
+});
+
+/**
+ * An extension may take over `services.jobs` (a fleet-wide scheduler). Core
+ * services register their jobs on it at start, so it must keep the built-in's
+ * first place in the layer.
+ */
+describe('PuterServer jobs replacement', () => {
+    it('lets an extension service registered as `jobs` replace the built-in and keep its first position', async () => {
+        const registered: string[] = [];
+        class ReplacementJobs extends JobScheduler {
+            override register(name: string, definition: JobDefinition) {
+                registered.push(name);
+                super.register(name, definition);
+            }
+        }
+        const started = vi.spyOn(JobScheduler.prototype, 'onServerStart');
+        extension.registerService('jobs', ReplacementJobs as never);
+        let server: PuterServer | undefined;
+        try {
+            server = await setupTestServer();
+            const { jobs } = server.services;
+            expect(jobs).toBeInstanceOf(ReplacementJobs);
+            expect(Object.keys(server.services)[0]).toBe('jobs');
+            expect(extension.import('service').jobs).toBeInstanceOf(
+                ReplacementJobs,
+            );
+            // Only the replacement gets lifecycle hooks.
+            expect(started.mock.contexts).toEqual([jobs]);
+            expect(registered).toContain('auth.sessions.purge');
+        } finally {
+            delete (extensionStore.services as Record<string, unknown>).jobs;
+            started.mockRestore();
+            await server?.shutdown();
+        }
+    });
+});
+
 /**
  * Each layer's `onServerShutdown` has to run while the layers beneath it (the
  * ones it writes through) are still up, so teardown goes top-down: drivers,

src/backend/server.ts

@@ -36,6 +36,10 @@ import { createAuthProbe } from './core/http/middleware/authProbe';
 import { createRequestContextMiddleware } from './core/http/middleware/requestContext';
 import { createFingerprintMiddleware } from './core/http/middleware/fingerprint';
 import { createErrorHandler } from './core/http/middleware/errorHandler';
+import {
+    createLargeJsonBodyParser,
+    DEFAULT_JSON_LIMIT_BYTES,
+} from './core/http/middleware/largeJsonBody';
 import { actorUid, type Actor } from './core/actor';
 import { isHttpError } from './core/http/HttpError';
 import {
@@ -117,6 +121,7 @@ import type {
     IConfig,
     LayerInstances,
     PagerSeverity,
+    ServerDrainInfo,
     WithControllerRegistration,
     WithLifecycle,
 } from './types';
@@ -124,6 +129,33 @@ import type {
 /** Idle keep-alive timeout used when `keep_alive_timeout` is unset. */
 const DEFAULT_KEEP_ALIVE_TIMEOUT = 920_000;
 
+/**
+ * AI request routes, which take bodies up to the size Claude's Messages API
+ * accepts from a signed-in caller. puter.ai.chat goes through `/drivers/call`.
+ */
+const LARGE_JSON_ROUTES = [
+    '/drivers/call',
+    '/puterai/openai/v1/chat/completions',
+    '/puterai/openai/v1/completions',
+    '/puterai/openai/v1/responses',
+    '/puterai/anthropic/v1/messages',
+    '/puterai/anthropic/v1/messages/count_tokens',
+];
+const LARGE_JSON_LIMIT_BYTES = 128 * 1024 * 1024;
+/** Two max-size bodies per process. */
+const LARGE_JSON_BUDGET_BYTES = 2 * LARGE_JSON_LIMIT_BYTES;
+
+/**
+ * Through `/drivers/call`, only chat takes a large body; other drivers keep the
+ * default limit. Keyed on the body rather than the path, which routing matches
+ * loosely (case, trailing slash). Wire API bodies carry no `interface`.
+ */
+const acceptsLargeJson = (req: Request): boolean => {
+    const iface = (req.body as { interface?: unknown } | null | undefined)
+        ?.interface;
+    return iface === undefined || iface === 'puter-chat-completion';
+};
+
 /** Route options a claimed subdomain's routes may use. */
 const CLAIMED_ROUTE_OPTIONS = new Set([
     'subdomain',
@@ -524,10 +556,34 @@ export class PuterServer {
             // body-parser hands over a fresh buffer and never writes to it.
             (req as { rawBody?: Buffer }).rawBody = buf;
         };
-        this.#app.use(express.json({ limit: '50mb', verify: captureRawBody }));
+        const authService = this.services.auth as AuthService | undefined;
+        const authProbe = authService
+            ? createAuthProbe({
+                  authService,
+                  cookieName: this.#config.cookie_name,
+              })
+            : undefined;
+        if (authProbe) {
+            this.#app.post(
+                LARGE_JSON_ROUTES,
+                createLargeJsonBodyParser({
+                    authProbe,
+                    limitBytes: LARGE_JSON_LIMIT_BYTES,
+                    budgetBytes: LARGE_JSON_BUDGET_BYTES,
+                    perUserBytes: LARGE_JSON_LIMIT_BYTES,
+                    accepts: acceptsLargeJson,
+                }),
+            );
+        }
         this.#app.use(
             express.json({
-                limit: '50mb',
+                limit: DEFAULT_JSON_LIMIT_BYTES,
+                verify: captureRawBody,
+            }),
+        );
+        this.#app.use(
+            express.json({
+                limit: DEFAULT_JSON_LIMIT_BYTES,
                 type: (req) =>
                     req.headers['content-type'] === 'text/plain;actually=json',
                 verify: captureRawBody,
@@ -541,14 +597,8 @@ export class PuterServer {
         this.#app.use(express.urlencoded({ extended: true, limit: '100kb' }));
 
         // -- Auth probe ----------------------------------------------
-        const authService = this.services.auth as AuthService | undefined;
-        if (authService) {
-            this.#app.use(
-                createAuthProbe({
-                    authService,
-                    cookieName: this.#config.cookie_name,
-                }),
-            );
+        if (authProbe) {
+            this.#app.use(authProbe);
         }
 
         // -- Request fingerprints ------------------------------------
@@ -1693,7 +1743,12 @@ export class PuterServer {
      */
     async #runHook(
         order: 'bottomUp' | 'topDown',
-        hook: 'onServerStart' | 'onServerPrepareShutdown' | 'onServerShutdown',
+        hook:
+            | 'onServerStart'
+            | 'onServerDrainStart'
+            | 'onServerPrepareShutdown'
+            | 'onServerShutdown',
+        info?: ServerDrainInfo,
     ) {
         const layers = [
             this.clients,
@@ -1705,25 +1760,33 @@ export class PuterServer {
         if (order === 'topDown') layers.reverse();
         for (const layer of layers) {
             for (const instance of Object.values(layer) as WithLifecycle[]) {
-                await instance[hook]?.();
+                await instance[hook]?.(info);
             }
         }
     }
 
-    #prepareShutdownHooksRan = false;
+    #prepareShutdownHooks: Promise<void> | null = null;
 
     /**
      * Run every layer's `onServerPrepareShutdown` exactly once, whichever of
-     * `prepareShutdown()` / `shutdown()` gets there first.
+     * `prepareShutdown()` / `shutdown()` gets there first; the other waits for
+     * that run to finish.
      */
-    async #runPrepareShutdownHooks() {
-        if (this.#prepareShutdownHooksRan) return;
-        this.#prepareShutdownHooksRan = true;
-        await this.#runHook('bottomUp', 'onServerPrepareShutdown');
+    #runPrepareShutdownHooks(): Promise<void> {
+        this.#prepareShutdownHooks ??= this.#runHook(
+            'bottomUp',
+            'onServerPrepareShutdown',
+        );
+        return this.#prepareShutdownHooks;
     }
 
-    async prepareShutdown() {
+    async prepareShutdown(info: ServerDrainInfo = {}) {
         if (this.#server) {
+            try {
+                await this.#runHook('bottomUp', 'onServerDrainStart', info);
+            } catch (err) {
+                console.error('[server] onServerDrainStart failed', err);
+            }
             this.#server.close(async () => {
                 console.log(
                     'PuterServer has stopped accepting new connections',

src/backend/services/auth/AuthService.test.ts

@@ -24,6 +24,7 @@ import { makeActor, type Actor } from '../../core/actor.js';
 import { PuterServer } from '../../server.js';
 import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js';
 import { setupTestServer } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import { generateDefaultFsentries } from '../../util/userProvisioning.js';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS } from '../permission/consts.js';
@@ -2041,6 +2042,45 @@ describe('AuthService (integration)', () => {
             );
             expect((await authService.authenticate(token)).actor).toBeTruthy();
         });
+
+        it('records no activity for an events handler`s calls', async () => {
+            const user = await makeUser();
+            const app = await server.stores.app.create(
+                {
+                    name: `ev-app-${uuidv4()}`,
+                    title: 'Events app',
+                    index_url: `https://${uuidv4()}.example.com/`,
+                },
+                { ownerUserId: user.id },
+            );
+            const handler = await authService.createWorkerAppToken(
+                makeActor({ user }),
+                app.uid,
+                EVENTS_WORKER_SESSION_NAME,
+                { handlerDepth: 1, expiresInSeconds: 900 },
+            );
+            const interactive = await authService.getUserAppToken(
+                makeActor({ user }),
+                app.uid,
+            );
+
+            const touch = vi.spyOn(server.stores.session, 'touch');
+            try {
+                const ctx = { ip: '9.9.9.9', userAgent: 'worker-runtime' };
+                expect(
+                    (await authService.authenticate(handler, ctx)).actor,
+                ).toBeTruthy();
+                expect(touch).not.toHaveBeenCalled();
+
+                // The same app in a tab is the user being active.
+                expect(
+                    (await authService.authenticate(interactive, ctx)).actor,
+                ).toBeTruthy();
+                expect(touch).toHaveBeenCalledTimes(1);
+            } finally {
+                touch.mockRestore();
+            }
+        });
     });
 
     describe('deleting worker sessions', () => {
@@ -2722,6 +2762,48 @@ describe('AuthService (integration)', () => {
                 authService.subdomainOwnerIdFromOrigin('not-a-url'),
             ).resolves.toBeNull();
         });
+
+        it('resolves a custom domain to its site owner', async () => {
+            const user = await makeUser();
+            const subdomain = `own-${Math.random().toString(36).slice(2, 10)}`;
+            await server.stores.subdomain.create({
+                userId: user.id,
+                subdomain,
+            });
+            const host = `cd-${uuidv4().slice(0, 8)}.example.com`;
+            const resolveHost = (
+                _key: unknown,
+                data: { host: string; result: { subdomain: string | null } },
+            ) => {
+                if (data.host === host) data.result.subdomain = subdomain;
+            };
+            const config = (
+                authService as unknown as { config: Record<string, unknown> }
+            ).config;
+            config.custom_domains_enabled = true;
+            server.clients.event.on('site.resolveHost', resolveHost);
+            try {
+                await expect(
+                    authService.subdomainOwnerIdFromOrigin(`https://${host}`),
+                ).resolves.toBe(user.id);
+                // A private app can't take another user's custom domain.
+                await expect(
+                    authService.originHeldByOtherUser(
+                        `https://${host}/`,
+                        user.id + 1,
+                    ),
+                ).resolves.toBe(true);
+                await expect(
+                    authService.originHeldByOtherUser(
+                        `https://${host}/`,
+                        user.id,
+                    ),
+                ).resolves.toBe(false);
+            } finally {
+                server.clients.event.off('site.resolveHost', resolveHost);
+                delete config.custom_domains_enabled;
+            }
+        });
     });
 
     describe('app origin blocklist enforcement', () => {
@@ -3967,7 +4049,6 @@ describe('AuthService (integration)', () => {
     });
 
     describe('purgeEndedSessions', () => {
-        const GATE_KEY = 'sessions:purge:gate';
         const DAY = 24 * 60 * 60;
 
         const makeSession = async (
@@ -3996,8 +4077,15 @@ describe('AuthService (integration)', () => {
             return rows.length > 0;
         };
 
+        const idOf = async (uuid: string) => {
+            const [row] = await server.clients.db.read(
+                'SELECT `id` FROM `sessions` WHERE `uuid` = ?',
+                [uuid],
+            );
+            return Number(row.id);
+        };
+
         it('deletes rows ended past their retention and keeps the rest', async () => {
-            await server.clients.redis.del(GATE_KEY);
             const user = await makeUser();
             const id = user.id;
             const oldRevoked = await makeSession(id, 'asset', 'revoked_at', 31);
@@ -4019,7 +4107,7 @@ describe('AuthService (integration)', () => {
             const live = await makeSession(id, 'web');
 
             expect(
-                await authService.purgeEndedSessions(),
+                await authService.purgeEndedSessions(testJobContext()),
             ).toBeGreaterThanOrEqual(3);
 
             expect(await exists(oldRevoked)).toBe(false);
@@ -4031,34 +4119,84 @@ describe('AuthService (integration)', () => {
             expect(await exists(live)).toBe(true);
         });
 
-        it('skips while another run holds the gate', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            await server.clients.redis.set(GATE_KEY, 'other-node');
+        it('starts from the saved cursor and clears it once the walk is done', async () => {
             const user = await makeUser();
-            const oldRevoked = await makeSession(
-                user.id,
-                'asset',
-                'revoked_at',
-                31,
-            );
+            const below = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const above = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const ctx = testJobContext({ afterId: await idOf(below) });
 
-            expect(await authService.purgeEndedSessions()).toBeNull();
-            expect(await exists(oldRevoked)).toBe(true);
-            await server.clients.redis.del(GATE_KEY);
+            expect(
+                await authService.purgeEndedSessions(ctx),
+            ).toBeGreaterThanOrEqual(1);
+            expect(await exists(below)).toBe(true);
+            expect(await exists(above)).toBe(false);
+            expect(ctx.saved).toEqual([null]);
         });
 
-        it('frees the gate when the walk fails', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            const spy = vi
-                .spyOn(server.clients.db, 'read')
-                .mockRejectedValueOnce(new Error('db down'));
+        it('clears its own checkpoints once a walk completes, so the next run starts at zero', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 11,
+            );
+            const ranges: number[] = [];
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    return 0;
+                });
             try {
-                await expect(authService.purgeEndedSessions()).rejects.toThrow(
-                    'db down',
+                const first = testJobContext();
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(11);
+                // Checkpointed at ten ranges, then cleared at the end.
+                expect(first.saved).toEqual([{ afterId: SPAN * 10 }, null]);
+
+                ranges.length = 0;
+                await authService.purgeEndedSessions(
+                    testJobContext(first.saved.at(-1)),
                 );
-                expect(await server.clients.redis.get(GATE_KEY)).toBeNull();
+                expect(ranges[0]).toBe(0);
+                expect(ranges).toHaveLength(11);
             } finally {
-                spy.mockRestore();
+                purge.mockRestore();
+                vi.restoreAllMocks();
+            }
+        });
+
+        it('stops between ranges once aborted, and the next run resumes at the saved cursor', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 30,
+            );
+            const ranges: number[] = [];
+            const first = testJobContext();
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    // Aborted mid-walk, as a lost lease or a shutdown would.
+                    if (ranges.length === 12) first.abort();
+                    return 0;
+                });
+            try {
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(12);
+                // Saved every 10 ranges, and where it stopped.
+                expect(first.saved).toEqual([
+                    { afterId: SPAN * 10 },
+                    { afterId: SPAN * 12 },
+                ]);
+
+                ranges.length = 0;
+                const next = testJobContext(first.saved.at(-1));
+                await authService.purgeEndedSessions(next);
+                expect(ranges[0]).toBe(SPAN * 12);
+                expect(ranges.at(-1)).toBe(SPAN * 29);
+                expect(next.saved.at(-1)).toBeNull();
+            } finally {
+                purge.mockRestore();
+                vi.restoreAllMocks();
             }
         });
     });

src/backend/services/auth/AuthService.ts

@@ -28,6 +28,7 @@ import {
 import { HttpError } from '../../core/http/HttpError.js';
 import { WEB_AND_EXTENSION_PROTOCOLS } from '../../util/validation.js';
 import { awaitLater } from '../../util/concurrency.js';
+import { appGenerationStart } from '../../util/appGeneration.js';
 import {
     ASSET_WINDOW_SECONDS,
     GODMODE_TOKEN_MAX_AGE_SECONDS,
@@ -39,11 +40,13 @@ import type { UserRow } from '../../stores/user/UserStore';
 import type { LayerInstances } from '../../types';
 import { sessionCookieFlags } from '../../util/cookieFlags.js';
 import { isGodmodeApp } from '../../util/godmodeApps.js';
+import { resolveCustomHostSubdomain } from '../../util/hostedAppBacking.js';
 import { Span } from '../../util/span.js';
 import type { puterServices } from '../index';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS, PERMISSION_MAX_LEN } from '../permission/consts';
 import type { GrantMeta } from '../permission/PermissionService.js';
+import { savedAfterId, type JobRunContext } from '../jobs/JobScheduler.js';
 import { PuterService } from '../types';
 import type {
     AccessTokenPayload,
@@ -56,22 +59,20 @@ import type {
 const APP_ORIGIN_UUID_NAMESPACE = '33de3768-8ee0-43e9-9e73-db192b97a5d8';
 // Successor uids an origin can derive after its earlier apps were repointed.
 const MAX_ORIGIN_UID_GENERATIONS = 8;
-// Allowed gap between a session's creation time (server clock) and its app's
-// (database clock) before the session counts as older than the app.
-const APP_SESSION_CLOCK_SKEW_SECONDS = 60;
 
 // Revoked and expired sessions are kept this long, so a stale token still gets
 // `session_revoked`/`session_expired` rather than `invalid`, then deleted. Web
 // rows double as sign-in history, so they stay longer.
 const SESSION_RETENTION_SECONDS = 30 * 24 * 60 * 60;
 const WEB_SESSION_RETENTION_SECONDS = 365 * 24 * 60 * 60;
-// Every node checks hourly; the gate lets one node through per week.
-const SESSION_PURGE_CHECK_MS = 60 * 60 * 1000;
-const SESSION_PURGE_GATE_KEY = 'sessions:purge:gate';
-const SESSION_PURGE_GATE_SECONDS = 7 * 24 * 60 * 60;
+const SESSION_PURGE_JOB = 'auth.sessions.purge';
+const SESSION_PURGE_EVERY_MS = 7 * 24 * 60 * 60 * 1000;
+const SESSION_PURGE_LEASE_MS = 30 * 60 * 1000;
 // Ids per DELETE, with a pause between, so replicas keep up.
 const SESSION_PURGE_SPAN = 5000;
 const SESSION_PURGE_PAUSE_MS = 50;
+/** Ranges walked between saves of the cursor. */
+const SESSION_PURGE_SAVE_EVERY = 10;
 
 const nowSeconds = (): number => Math.floor(Date.now() / 1000);
 
@@ -139,11 +140,17 @@ export interface AuthResult {
 export class AuthService extends PuterService {
     declare protected services: LayerInstances<typeof puterServices>;
 
-    #sessionPurge: ReturnType<typeof setInterval> | null = null;
-    #stopping = false;
-
     override onServerStart(): void {
-        this.#armSessionPurge();
+        this.services.jobs.register(SESSION_PURGE_JOB, {
+            every: SESSION_PURGE_EVERY_MS,
+            scope: 'fleet',
+            nearPrimary: true,
+            leaseMs: SESSION_PURGE_LEASE_MS,
+            run: async (ctx) => {
+                const purged = await this.purgeEndedSessions(ctx);
+                console.log(`[auth] purged ${purged} ended sessions`);
+            },
+        });
 
         // Users implicitly hold read access to their own email — needed for
         // any permission-gated path that asks for `user:<uuid>:email:read`
@@ -165,85 +172,54 @@ export class AuthService extends PuterService {
         });
     }
 
-    override onServerPrepareShutdown(): void {
-        this.#stopping = true;
-        if (this.#sessionPurge) clearInterval(this.#sessionPurge);
-        this.#sessionPurge = null;
-    }
-
     // -- Session purge -----------------------------------------------
 
     /**
      * Delete sessions revoked or expired longer ago than their retention,
-     * walking the table by id. Returns the rows deleted, or null when another
-     * node already holds this week's gate.
+     * walking the table by id from the cursor an interrupted run saved. Returns
+     * the rows deleted; stops early once `signal` aborts.
      */
-    async purgeEndedSessions(): Promise<number | null> {
-        const claimed = await this.clients.redis.set(
-            SESSION_PURGE_GATE_KEY,
-            String(Date.now()),
-            'EX',
-            SESSION_PURGE_GATE_SECONDS,
-            'NX',
-        );
-        if (claimed !== 'OK') return null;
-
+    async purgeEndedSessions({
+        signal,
+        state,
+        saveState,
+    }: JobRunContext): Promise<number> {
         const now = nowSeconds();
         const cutoffs = {
             cutoff: now - SESSION_RETENTION_SECONDS,
             webCutoff: now - WEB_SESSION_RETENTION_SECONDS,
         };
+        const maxId = await this.stores.session.maxId();
+        let afterId = savedAfterId(state);
         let purged = 0;
-        let finished = false;
-        try {
-            const maxId = await this.stores.session.maxId();
-            for (
-                let afterId = 0;
-                afterId < maxId;
-                afterId += SESSION_PURGE_SPAN
-            ) {
-                if (this.#stopping) return purged;
-                purged += await this.stores.session.purgeEndedInRange(
-                    afterId,
-                    SESSION_PURGE_SPAN,
-                    cutoffs,
-                );
-                await new Promise((resolve) =>
-                    setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
-                );
-            }
-            finished = true;
-            return purged;
-        } finally {
-            // A walk cut short frees the gate, so another node restarts it on
-            // its next check instead of next week.
-            if (!finished) {
+        let unsaved = 0;
+        while (afterId < maxId) {
+            if (signal.aborted) {
+                // Kept for the next run if this one still holds the job.
                 try {
-                    await this.clients.redis.del(SESSION_PURGE_GATE_KEY);
+                    await saveState({ afterId });
                 } catch {
-                    // The gate expires on its own.
+                    // Lost the job; the next run redoes these ranges.
                 }
+                return purged;
             }
-        }
-    }
-
-    #armSessionPurge(): void {
-        const timer = setInterval(() => {
-            void this.#runSessionPurge();
-        }, SESSION_PURGE_CHECK_MS);
-        timer.unref?.();
-        this.#sessionPurge = timer;
-    }
-
-    async #runSessionPurge(): Promise<void> {
-        try {
-            const purged = await this.purgeEndedSessions();
-            if (purged !== null) {
-                console.log(`[auth] purged ${purged} ended sessions`);
+            purged += await this.stores.session.purgeEndedInRange(
+                afterId,
+                SESSION_PURGE_SPAN,
+                cutoffs,
+            );
+            afterId += SESSION_PURGE_SPAN;
+            if (++unsaved >= SESSION_PURGE_SAVE_EVERY) {
+                await saveState({ afterId });
+                unsaved = 0;
             }
-        } catch (err) {
-            console.warn('[auth] session purge failed', err);
+            await new Promise((resolve) =>
+                setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
+            );
         }
+        // Clears any checkpoint, this run's included, so next week starts at 0.
+        await saveState(null);
+        return purged;
     }
 
     // -- Public API --------------------------------------------------
@@ -561,7 +537,7 @@ export class AuthService extends PuterService {
                 appUid,
                 workerName,
                 auth_id,
-                notBefore: this.#appSessionsNotBefore(app),
+                notBefore: appGenerationStart(app),
             },
         );
         if (!session) {
@@ -952,13 +928,18 @@ export class AuthService extends PuterService {
         for (const entry of revoked) this.#announceRevocation(entry);
     }
 
-    /** Delete the owner's sessions for a worker that no longer exists. */
+    /**
+     * Delete the owner's sessions for a worker that no longer exists, or, with
+     * `exceptAppUid`, the ones a worker rebound to that app (`null`: to none)
+     * no longer runs on.
+     */
     async deleteWorkerSessionsByName(
         workerName: string,
         userId: number,
+        { exceptAppUid }: { exceptAppUid?: string | null } = {},
     ): Promise<void> {
         if (!workerName || !userId) return;
-        await this.#deleteWorkerSessions({ workerName, userId });
+        await this.#deleteWorkerSessions({ workerName, userId, exceptAppUid });
     }
 
     /**
@@ -1014,6 +995,7 @@ export class AuthService extends PuterService {
         workerName?: string;
         userId?: number;
         appUids?: string[];
+        exceptAppUid?: string | null;
     }): Promise<void> {
         const deleted = (await this.stores.session.deleteWorkerSessions(
             filter,
@@ -1360,13 +1342,13 @@ export class AuthService extends PuterService {
      * Resolve the user who owns the hosted subdomain `origin` points at.
      * Returns the `subdomains` row's `user_id` when the origin's host sits
      * under a configured hosting domain (`static_hosting_domain(_alt)` /
-     * `private_app_hosting_domain(_alt)`) and the subdomain is registered; null
-     * for external origins, apex hosting-domain hosts, and unknown subdomains.
-     * Used to stamp the site owner as the creator of origin-bootstrap app
-     * rows.
+     * `private_app_hosting_domain(_alt)`), or is a custom domain serving a
+     * site, and the subdomain is registered; null for external origins, apex
+     * hosting-domain hosts, and unknown subdomains. Used to stamp the site
+     * owner as the creator of origin-bootstrap app rows.
      */
     async subdomainOwnerIdFromOrigin(origin: string): Promise<number | null> {
-        const subdomain = this.#hostedSubdomainForOrigin(origin);
+        const subdomain = await this.#siteSubdomainForOrigin(origin);
         if (!subdomain) return null;
         const row = await this.stores.subdomain.getBySubdomain(subdomain);
         return positiveUserId(row?.user_id);
@@ -1385,7 +1367,9 @@ export class AuthService extends PuterService {
     ): Promise<boolean> {
         const origin = await this.#appOriginFor(indexUrl);
         if (!origin) return false;
-        const siteOwnerId = await this.#siteOwnerIdForOrigin(origin);
+        const siteOwnerId = await this.#siteOwnerIdForSubdomain(
+            await this.#siteSubdomainForOrigin(origin),
+        );
         if (siteOwnerId !== null) return siteOwnerId !== ownerUserId;
 
         const path = new URL(indexUrl).pathname.toLowerCase();
@@ -1398,12 +1382,20 @@ export class AuthService extends PuterService {
         return holderId !== null && holderId !== ownerUserId;
     }
 
-    /**
-     * {@link subdomainOwnerIdFromOrigin}, confirmed against the primary on a
-     * miss so a site registered a moment ago isn't read as unregistered.
-     */
+    /** Owner of the hosted site `origin` is on; custom domains aren't asked. */
     async #siteOwnerIdForOrigin(origin: string): Promise<number | null> {
-        const subdomain = this.#hostedSubdomainForOrigin(origin);
+        return this.#siteOwnerIdForSubdomain(
+            this.#hostedSubdomainForOrigin(origin),
+        );
+    }
+
+    /**
+     * The site's owner, confirmed against the primary on a miss so a site
+     * registered a moment ago isn't read as unregistered.
+     */
+    async #siteOwnerIdForSubdomain(
+        subdomain: string | null,
+    ): Promise<number | null> {
         if (!subdomain) return null;
         const row =
             (await this.stores.subdomain.getBySubdomain(subdomain)) ??
@@ -1413,6 +1405,25 @@ export class AuthService extends PuterService {
         return positiveUserId(row?.user_id);
     }
 
+    /**
+     * Site `origin` serves: a hosted subdomain, or a custom domain's site.
+     * Custom domains cost a listener round trip, so identity resolution
+     * (`appUidFromOrigin` and its batch) stays on hosted names only.
+     */
+    async #siteSubdomainForOrigin(origin: string): Promise<string | null> {
+        const hosted = this.#hostedSubdomainForOrigin(origin);
+        if (hosted) return hosted;
+        const parsed = URL.parse(origin);
+        if (parsed?.protocol !== 'https:' && parsed?.protocol !== 'http:') {
+            return null;
+        }
+        return resolveCustomHostSubdomain(
+            parsed.hostname,
+            this.config,
+            this.clients.event,
+        );
+    }
+
     #hostedSubdomainForOrigin(origin: string): string | null {
         let parsed: URL;
         try {
@@ -1778,7 +1789,7 @@ export class AuthService extends PuterService {
             appUid,
             {
                 auth_id: this.#authIdFor(actor.user as UserRow),
-                notBefore: this.#appSessionsNotBefore(app),
+                notBefore: appGenerationStart(app),
             },
         );
 
@@ -1836,7 +1847,7 @@ export class AuthService extends PuterService {
                 appUid: app.uid,
                 expiresAt: nowSeconds() + GODMODE_TOKEN_WINDOW_SECONDS,
                 maxAgeSeconds: GODMODE_TOKEN_MAX_AGE_SECONDS,
-                label: app.title || app.name || null,
+                label: app.title || app.name!,
                 auth_id: authId,
             },
         );
@@ -1846,7 +1857,7 @@ export class AuthService extends PuterService {
                 legacyCode: 'internal_error',
             });
         }
-        if (minted.replaced) await this.#retireGodmodeToken(minted.replaced);
+        if (minted?.replaced) await this.#retireGodmodeToken(minted.replaced);
 
         const token = this.services.token.sign('auth', {
             type: 'access-token',
@@ -2655,14 +2666,17 @@ export class AuthService extends PuterService {
         // uid returns with the next app on that origin. A session older than
         // the app was made for the earlier one. Checked before `touch()` so a
         // rejected token doesn't slide the old session's expiry.
-        const notBefore = this.#appSessionsNotBefore(app);
+        const notBefore = appGenerationStart(app);
         const createdAt = Number(session.created_at);
         if (notBefore !== null && createdAt > 0 && createdAt < notBefore) {
             return { reauth: { reason: 'session_revoked' } };
         }
         if (!(await deployed)) return { invalid: true };
 
-        void this.#touchSession(session.uuid, user.id, ctx);
+        // An events handler's calls are background work, not the user being
+        // active, and its session never expires, so they record no activity.
+        if (decoded.worker_name !== EVENTS_WORKER_SESSION_NAME)
+            void this.#touchSession(session.uuid, user.id, ctx);
 
         const actor = this.#buildAppUnderUserActor(user, app, session);
         this.#applyHandlerDepth(actor, decoded);
@@ -2684,19 +2698,6 @@ export class AuthService extends PuterService {
         return block.blocked ? { reason: block.reason } : null;
     }
 
-    /**
-     * Unix seconds before which an app or worker session for `app` belongs to
-     * an earlier app with the same uid; null when the creation time is
-     * unknown.
-     */
-    #appSessionsNotBefore(
-        app: { created_epoch?: unknown } | null | undefined,
-    ): number | null {
-        const created = Number(app?.created_epoch);
-        if (!Number.isFinite(created) || created <= 0) return null;
-        return created - APP_SESSION_CLOCK_SKEW_SECONDS;
-    }
-
     #createdBefore(row: SessionRow | null, notBefore: number | null): boolean {
         if (notBefore === null || !row) return false;
         const createdAt = Number(row.created_at);
@@ -2772,7 +2773,7 @@ export class AuthService extends PuterService {
             if (blocked) return { blocked };
             // As with app sessions: a token minted under an earlier app that
             // held this uid doesn't authenticate as the current one.
-            const notBefore = this.#appSessionsNotBefore(app);
+            const notBefore = appGenerationStart(app);
             if (
                 this.#createdBefore(session, notBefore) ||
                 this.#createdBefore(parentSession, notBefore)

src/backend/services/email/mailbox.test.ts

@@ -1,9 +1,17 @@
 import { Readable } from 'node:stream';
 import { v4 as uuidv4 } from 'uuid';
-import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest';
+import { EVENTS_COALESCE_WINDOW_MS } from '../../controllers/events/limits.js';
+import type { Actor } from '../../core/actor.js';
+import { runWithContext } from '../../core/context.js';
 import type { PuterServer } from '../../server.js';
 import type { UserRow } from '../../stores/user/UserStore.js';
-import { createTestUser, setupTestServer } from '../../testUtil.js';
+import {
+    createTestUser,
+    setupTestServer,
+    type TestUserCredentials,
+} from '../../testUtil.js';
+import type { DeliveryEnvelope } from '../events/EventsService.js';
 import {
     MAIL_CONTENT_TYPE,
     copyIntoInbox,
@@ -28,6 +36,7 @@ const UUID_V7 =
 // which methods were called.
 let server: PuterServer;
 let dan: UserRow;
+let danCredentials: TestUserCredentials;
 
 // `frankfurt` is wired to the same bucket the test server already has, so a
 // message filed for a Frankfurt account lands somewhere real and the region it
@@ -42,8 +51,15 @@ beforeAll(async () => {
                 bucketRegion: FRANKFURT_REGION,
             },
         },
+        events: { enabled: true },
+        // Seeded accounts have no email, which plans read as a temporary
+        // account that holds no persistent subscriptions.
+        unlimitedMetering: true,
     } as never);
-    await createTestUser(server, { username: 'dan', password: 'secret123!' });
+    danCredentials = await createTestUser(server, {
+        username: 'dan',
+        password: 'secret123!',
+    });
     dan = (await server.stores.user.getByUsername('dan')) as UserRow;
 });
 
@@ -228,3 +244,83 @@ describe('copyIntoInbox', () => {
         expect(copied.size).toBe(source.size);
     });
 });
+
+describe('the filesystem events filing raises', () => {
+    let actor: Actor;
+    let subId: string;
+    const delivered: DeliveryEnvelope[] = [];
+
+    beforeAll(async () => {
+        actor = (await server.services.auth.authenticate(danCredentials.token))
+            .actor!;
+        ({
+            sub: { subId },
+        } = await server.services.events.subscribeDurable(actor, {
+            subject: `fs:${mailboxPath('dan')}`,
+        }));
+        server.services.events.onDelivered = (envelope) =>
+            delivered.push(envelope);
+    });
+
+    const deliveredAt = (path: string) =>
+        vi.waitFor(
+            () => {
+                const found = delivered.find(
+                    (envelope) =>
+                        envelope.subId === subId &&
+                        (envelope.event as { path?: string }).path === path,
+                );
+                expect(found).toBeDefined();
+                return found!.event as { op: string; self: boolean };
+            },
+            { timeout: EVENTS_COALESCE_WINDOW_MS * 12, interval: 25 },
+        );
+
+    test("is not the owner's own change, even inside the owner's request", async () => {
+        const entry = await runWithContext({ actor }, () =>
+            storeInboxMessage(server.services.fs, dan, {
+                subject: 'to myself',
+                content: Buffer.from('raw'),
+                size: 3,
+            }),
+        );
+
+        expect(await deliveredAt(entry.path)).toMatchObject({
+            op: 'add',
+            self: false,
+        });
+    });
+
+    test('nor is a message copied in', async () => {
+        const source = await storeInboxMessage(server.services.fs, dan, {
+            content: Buffer.from('original'),
+            size: 8,
+            day: '2021-01-01',
+            name: `source-${uuidv4()}`,
+        });
+
+        const copied = await runWithContext({ actor }, () =>
+            copyIntoInbox(server.services.fs, dan, {
+                source,
+                name: `copied-${uuidv4()}`,
+                day: '2021-01-02',
+            }),
+        );
+
+        expect((await deliveredAt(copied.path)).self).toBe(false);
+    });
+
+    test("while the owner's own write in the same request still is", async () => {
+        const { fsEntry } = await runWithContext({ actor }, () =>
+            server.services.fs.write(dan.id, {
+                fileMetadata: {
+                    path: `${mailboxPath('dan')}/note-${uuidv4()}.txt`,
+                    size: 1,
+                },
+                fileContent: Buffer.from('x'),
+            }),
+        );
+
+        expect((await deliveredAt(fsEntry.path)).self).toBe(true);
+    });
+});

src/backend/services/email/mailbox.ts

@@ -18,6 +18,7 @@
  */
 
 import type { Readable } from 'node:stream';
+import { Context, runInDerivedContext } from '../../core/context.js';
 import type { FSService } from '../fs/FSService.js';
 import { resolveHomeRegion } from '../fs/homeRegion.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
@@ -133,6 +134,17 @@ export interface InboxMessage {
     name?: string;
 }
 
+/**
+ * Runs a mailbox write as a delivery to the owner rather than a write by
+ * whoever's request it is in, so filing is never the holder's own change
+ * (`self`), even for mail the owner sent to itself.
+ */
+const asDelivery = <T>(file: () => Promise<T>): Promise<T> =>
+    runInDerivedContext(() => {
+        Context.set('platformWrite', true);
+        return file();
+    });
+
 /**
  * File one message in the owner's inbox. Written into the owner's home region
  * rather than wherever the request landed, so the mailbox reads locally.
@@ -144,22 +156,24 @@ export async function storeInboxMessage(
 ): Promise<FSEntry> {
     const day = message.day ?? mailDay();
     const name = message.name ?? mailObjectName(message.subject);
-    const { fsEntry } = await fs.write(
-        owner.id,
-        {
-            fileMetadata: {
-                path: `${mailFolderPath(owner.username, 'objects', day)}/${name}`,
-                size: message.size,
-                contentType: MAIL_CONTENT_TYPE,
-                createMissingParents: true,
-                overwrite: false,
-                dedupeName: false,
+    const { fsEntry } = await asDelivery(() =>
+        fs.write(
+            owner.id,
+            {
+                fileMetadata: {
+                    path: `${mailFolderPath(owner.username, 'objects', day)}/${name}`,
+                    size: message.size,
+                    contentType: MAIL_CONTENT_TYPE,
+                    createMissingParents: true,
+                    overwrite: false,
+                    dedupeName: false,
+                },
+                fileContent: message.content,
             },
-            fileContent: message.content,
-        },
-        undefined,
-        undefined,
-        resolveHomeRegion(owner),
+            undefined,
+            undefined,
+            resolveHomeRegion(owner),
+        ),
     );
     return fsEntry;
 }
@@ -174,17 +188,19 @@ export async function copyIntoInbox(
     owner: UserRow,
     message: { source: FSEntry; name: string; day?: string },
 ): Promise<FSEntry> {
-    const parent = await fs.mkdir(owner.id, {
-        path: mailFolderPath(
-            owner.username,
-            'objects',
-            message.day ?? mailDay(),
-        ),
-        createMissingParents: true,
-    });
-    return await fs.copy(owner.id, {
-        source: message.source,
-        destinationParent: parent,
-        newName: message.name,
+    return asDelivery(async () => {
+        const parent = await fs.mkdir(owner.id, {
+            path: mailFolderPath(
+                owner.username,
+                'objects',
+                message.day ?? mailDay(),
+            ),
+            createMissingParents: true,
+        });
+        return await fs.copy(owner.id, {
+            source: message.source,
+            destinationParent: parent,
+            newName: message.name,
+        });
     });
 }

src/backend/services/events/EventsService.test.ts

@@ -24,6 +24,7 @@ import {
     EVENTS_COALESCE_WINDOW_MS,
     EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT,
     EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT,
+    EVENTS_KV_VALUE_DELIVERIES_PER_EVENT,
     EVENTS_KV_VALUE_MAX_BYTES,
     EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT,
     EVENTS_SUBSCRIBE_LIMIT,
@@ -205,6 +206,7 @@ let handles: Map<string, KvShareHandle>;
 
 const kvShareHandleStore = {
     getByHandle: async (handle: string) => handles.get(handle) ?? null,
+    predatesItsApp: async () => false,
     getLive: async (handle: string) => {
         const row = handles.get(handle);
         if (!row || row.revokedAt !== null) return null;
@@ -291,12 +293,14 @@ const buildService = (
     metered: MeteredLine[];
     forwarded: ForwardedCall[];
     eventBus: { on: ReturnType<typeof vi.fn>; emit: ReturnType<typeof vi.fn> };
+    jobs: { register: ReturnType<typeof vi.fn> };
 } => {
     const outbox: Array<{ socket?: string; envelope: DeliveryEnvelope }> = [];
     const counted: DeliveryEnvelope[] = [];
     const lines: MeteredLine[] = [];
     const forwardedCalls: ForwardedCall[] = [];
     const bus = { on: vi.fn(), emit: vi.fn() };
+    const jobs = { register: vi.fn() };
     const built = new EventsService(
         config,
         {
@@ -313,6 +317,7 @@ const buildService = (
             kvShareHandle: kvShareHandleStore,
         } as never,
         {
+            jobs,
             eventForward: {
                 // A deployment with no peers has nowhere to forward to, which
                 // is what every test here is.
@@ -374,6 +379,7 @@ const buildService = (
         metered: lines,
         forwarded: forwardedCalls,
         eventBus: bus,
+        jobs,
     };
 };
 
@@ -562,6 +568,52 @@ describe('the feature switch', () => {
         );
     });
 
+    it('hands its table sweeps to the job scheduler and arms only the pending sweep itself', () => {
+        const intervals = vi.spyOn(globalThis, 'setInterval');
+        const timeouts = vi.spyOn(globalThis, 'setTimeout');
+        let jobs: { register: ReturnType<typeof vi.fn> };
+        try {
+            ({ jobs } = buildService({ events: { enabled: true } } as IConfig));
+            // The pending sweep is the region's queue, so every node runs it.
+            expect(intervals.mock.calls.map(([, ms]) => ms)).toEqual([10_000]);
+            expect(timeouts).not.toHaveBeenCalled();
+        } finally {
+            intervals.mockRestore();
+            timeouts.mockRestore();
+        }
+        expect(
+            jobs.register.mock.calls.map(([name, job]) => ({
+                name,
+                every: job.every,
+                scope: job.scope,
+                nearPrimary: job.nearPrimary ?? false,
+                leaseMs: job.leaseMs,
+            })),
+        ).toEqual([
+            {
+                name: 'events.subscriptions.expirySweep',
+                every: 60 * 60_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: 15 * 60_000,
+            },
+            {
+                name: 'events.subscriptions.creditSweep',
+                every: 15 * 60_000,
+                scope: 'fleet',
+                nearPrimary: false,
+                leaseMs: 10 * 60_000,
+            },
+        ]);
+    });
+
+    it('registers no job when off', () => {
+        const { jobs } = buildService({
+            events: { enabled: false },
+        } as IConfig);
+        expect(jobs.register).not.toHaveBeenCalled();
+    });
+
     it('spends nothing at all on a dispatch when off', async () => {
         const { service: off } = buildService({
             events: { enabled: false },
@@ -2436,20 +2488,20 @@ describe('delivering a kv change', () => {
         expect(ownerPlanActors).toEqual([]);
     });
 
-    it('omits values and gaps the 129th free-tier KV listener', async () => {
+    it('keeps values on the delivered free-tier KV listeners and gaps the 129th', async () => {
         vi.useFakeTimers();
         await seedKvSubscriptions(129, { includeValue: true });
 
-        await dispatchKv(['cart'], { values: ['re-read'] });
+        await dispatchKv(['cart'], { values: ['kept'] });
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
         const deliveries = sent.filter((one) => one.envelope.event.op === 'set');
         const gaps = sent.filter((one) => one.envelope.event.op === 'gap');
         expect(deliveries).toHaveLength(128);
         expect(gaps).toHaveLength(1);
-        expect(deliveries.every((one) => !('value' in one.envelope.event))).toBe(
-            true,
-        );
+        expect(
+            deliveries.every((one) => one.envelope.event.value === 'kept'),
+        ).toBe(true);
         expect(ownerPlanActors).toEqual([
             expect.objectContaining({
                 user: expect.objectContaining({
@@ -2474,9 +2526,10 @@ describe('delivering a kv change', () => {
         expect(sent).toHaveLength(
             EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.limit,
         );
-        expect(sent.every((one) => !('value' in one.envelope.event))).toBe(
-            true,
-        );
+        // Values ride on the first deliveries only.
+        expect(
+            sent.filter((one) => 'value' in one.envelope.event),
+        ).toHaveLength(EVENTS_KV_VALUE_DELIVERIES_PER_EVENT);
     });
 
     it('gaps the 513th paid-tier KV listener', async () => {
@@ -2512,7 +2565,7 @@ describe('delivering a kv change', () => {
         );
     });
 
-    it('omits a forwarded value when the KV filter evaluation stops early', async () => {
+    it('keeps a forwarded value on the deliveries made when the KV filter evaluation stops early', async () => {
         vi.useFakeTimers();
         await seedKvSubscriptions(128, { includeValue: true });
         await seedKvSubscriptions(
@@ -2542,9 +2595,111 @@ describe('delivering a kv change', () => {
 
         const deliveries = sent.filter((one) => one.envelope.event.op === 'set');
         expect(deliveries).toHaveLength(128);
-        expect(deliveries.every((one) => !('value' in one.envelope.event))).toBe(
-            true,
-        );
+        expect(
+            deliveries.every(
+                (one) => one.envelope.event.value === 'forwarded-re-read',
+            ),
+        ).toBe(true);
+    });
+
+    describe('shared by several holders', () => {
+        const valueOf = (subIdPart: string) =>
+            sent.filter(
+                (one) =>
+                    one.envelope.subId.includes(subIdPart) &&
+                    one.envelope.event.op === 'set',
+            );
+
+        it('serves the owner`s rows first and every holder before anyone`s second row', async () => {
+            vi.useFakeTimers();
+            ownerPlan = 'paid-plan';
+            const crowd = userId + 600;
+            const quiet = userId + 700;
+            await seedKvSubscriptions(
+                EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.limit + 50,
+                { includeValue: true, holderUserId: crowd, tag: 'crowd' },
+            );
+            await seedKvSubscriptions(1, {
+                includeValue: true,
+                holderUserId: quiet,
+                tag: 'quiet',
+            });
+            await seedKvSubscriptions(1, { includeValue: true, tag: 'owner' });
+
+            await dispatchKv(['cart'], { values: ['shared'] });
+            await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
+
+            const delivered = sent.filter(
+                (one) => one.envelope.event.op === 'set',
+            );
+            expect(delivered).toHaveLength(
+                EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.limit,
+            );
+            for (const tag of ['-owner-', '-quiet-']) {
+                expect(valueOf(tag)).toHaveLength(1);
+                expect(valueOf(tag)[0].envelope.event.value).toBe('shared');
+            }
+            expect(
+                delivered.filter((one) => 'value' in one.envelope.event),
+            ).toHaveLength(EVENTS_KV_VALUE_DELIVERIES_PER_EVENT);
+        });
+
+        it('keeps the owner first even past the filter-check limit', async () => {
+            vi.useFakeTimers();
+            const filters =
+                EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT.bySubscription[
+                    DEFAULT_FREE_SUBSCRIPTION
+                ];
+            await seedKvSubscriptions(filters, {
+                includeValue: true,
+                holderUserId: userId + 600,
+                match: 'elsewhere',
+                tag: 'crowd',
+            });
+            await seedKvSubscriptions(1, { includeValue: true, tag: 'owner' });
+
+            await dispatchKv(['cart'], { values: ['first'] });
+            await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
+
+            expect(valueOf('-owner-')).toHaveLength(1);
+            expect(valueOf('-owner-')[0].envelope.event.value).toBe('first');
+        });
+
+        it('starts each event`s turns at a different holder, so none is always past the cap', async () => {
+            vi.useFakeTimers();
+            const holders = 200;
+            for (let i = 0; i < holders; i++)
+                await seedKvSubscriptions(1, {
+                    holderUserId: userId + 1000 + i,
+                    tag: `holder${i}`,
+                });
+            const cap =
+                EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.bySubscription[
+                    DEFAULT_FREE_SUBSCRIPTION
+                ];
+
+            const reached = new Set<string>();
+            const random = vi.spyOn(Math, 'random');
+            try {
+                for (const start of [0, cap / holders]) {
+                    random.mockReturnValue(start);
+                    sent.length = 0;
+                    await dispatchKv(['cart']);
+                    await vi.advanceTimersByTimeAsync(
+                        EVENTS_COALESCE_WINDOW_MS + 1,
+                    );
+                    const delivered = sent.filter(
+                        (one) => one.envelope.event.op === 'set',
+                    );
+                    expect(delivered).toHaveLength(cap);
+                    for (const one of delivered)
+                        reached.add(one.envelope.subId);
+                }
+            } finally {
+                random.mockRestore();
+            }
+            expect(reached.size).toBe(holders);
+        });
     });
 
     it('leaves another app`s namespace alone', async () => {
@@ -2998,6 +3153,7 @@ describe('cross-user kv handles', () => {
             keyPrefix: PREFIX,
             permission,
             label: null,
+            notify: false,
             createdAt: 0,
             revokedAt: null,
             revokedBy: null,
@@ -3069,8 +3225,19 @@ describe('cross-user kv handles', () => {
         expect(wire).not.toContain(`u${userId}`);
     });
 
+    /** The handle at `read`, the lowest level that carries values. */
+    const mintReadHandle = () =>
+        mintHandle({
+            permission: kvSharePermission(
+                `user-${userId}`,
+                OWN_APP,
+                PREFIX,
+                'read',
+            ),
+        });
+
     it('hands a guest the value when it asked, keyed relative to the handle', async () => {
-        mintHandle();
+        mintReadHandle();
         vi.useFakeTimers();
         const asking = (
             await service.subscribe(actorFor(guestId), socketId, {
@@ -3095,6 +3262,23 @@ describe('cross-user kv handles', () => {
         expect(byId.get(silent.subId)).not.toHaveProperty('value');
     });
 
+    it('holds the value back from a guest whose share is at watch', async () => {
+        mintHandle();
+        vi.useFakeTimers();
+        const { sub } = await service.subscribe(actorFor(guestId), socketId, {
+            subject: `kv:${handle}:*`,
+            includeValue: true,
+        });
+
+        await dispatchKv([`${PREFIX}title`], { values: ['hello'] });
+        await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
+
+        expect(sent).toHaveLength(1);
+        expect(sent[0].envelope.subId).toBe(sub.subId);
+        expect(sent[0].envelope.event).toMatchObject({ key: 'title' });
+        expect(sent[0].envelope.event).not.toHaveProperty('value');
+    });
+
     it('delivers every key under the granted region', async () => {
         mintHandle();
         vi.useFakeTimers();
@@ -3112,7 +3296,7 @@ describe('cross-user kv handles', () => {
     });
 
     it('withholds a private key from the grantee, value and all', async () => {
-        mintHandle();
+        mintReadHandle();
         vi.useFakeTimers();
         const { sub } = await service.subscribe(actorFor(guestId), socketId, {
             subject: `kv:${handle}:*`,

src/backend/services/events/appDeletion.integration.test.ts

@@ -29,6 +29,7 @@
 import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { v4 as uuidv4 } from 'uuid';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import { EVENTS_WORKER_SESSION_NAME } from './workerRuntime.js';
@@ -327,7 +328,9 @@ describe('an app deleted while it has events state', () => {
         await env.server.stores.app.delete((row as { id: number }).id);
 
         const revoked =
-            await env.server.services.events.sweepStrandedWorkerSessions();
+            await env.server.services.events.sweepStrandedWorkerSessions(
+                testJobContext(),
+            );
         expect(revoked).toBeGreaterThanOrEqual(1);
 
         // `authenticate()` can no longer resolve the token at all — the app
@@ -366,9 +369,60 @@ describe('an app deleted while it has events state', () => {
             EVENTS_WORKER_SESSION_NAME,
         );
 
-        await env.server.services.events.sweepStrandedWorkerSessions();
+        await env.server.services.events.sweepStrandedWorkerSessions(
+            testJobContext(),
+        );
 
         const reauth = await env.server.services.auth.authenticate(token);
         expect(reauth).not.toHaveProperty('reauth');
     });
+
+    it('reaches a stranded session past one run`s two pages on the next run, then wraps', async () => {
+        const db = env.server.clients.db;
+        const now = Math.floor(Date.now() / 1000);
+        const meta = JSON.stringify({
+            worker_name: EVENTS_WORKER_SESSION_NAME,
+        });
+        const workerSession = (appUid: string) => ({
+            statement:
+                'INSERT INTO `sessions` (`uuid`, `user_id`, `meta`, `last_activity`, `created_at`, `kind`, `app_uid`) ' +
+                "VALUES (?, ?, ?, ?, ?, 'worker', ?)",
+            values: [uuidv4(), userId, meta, now, now, appUid],
+        });
+        // A run reads two pages of 500; these fill them, all still standing.
+        const live = Array.from(
+            { length: 1000 },
+            (_, i) => `app-live-${uuidv4().slice(0, 8)}-${i}`,
+        );
+        await db.batchWrite(
+            live.map((uid) => ({
+                statement:
+                    'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
+                values: [uid, uid, uid, `https://${uid}.example/`, userId],
+            })),
+        );
+        await db.batchWrite(live.map(workerSession));
+        const stranded = workerSession(`app-gone-${uuidv4()}`);
+        await db.batchWrite([stranded]);
+        const strandedRevoked = async () => {
+            const [row] = await db.pread(
+                'SELECT `revoked_at` FROM `sessions` WHERE `uuid` = ?',
+                [stranded.values[0]],
+            );
+            return row.revoked_at !== null;
+        };
+
+        const first = testJobContext();
+        await env.server.services.events.sweepStrandedWorkerSessions(first);
+        expect(await strandedRevoked()).toBe(false);
+        expect(first.saved.at(-1)).toMatchObject({
+            afterId: expect.any(Number),
+        });
+
+        const second = testJobContext(first.saved.at(-1));
+        await env.server.services.events.sweepStrandedWorkerSessions(second);
+        expect(await strandedRevoked()).toBe(true);
+        // The end of the table wraps the cursor back to the start.
+        expect(second.saved.at(-1)).toBeNull();
+    });
 });

src/backend/services/events/authorization.ts

@@ -240,16 +240,17 @@ export const unknownKvShareHandle = (handle: string): HttpError =>
         legacyCode: 'subject_does_not_exist',
     });
 
+/** Whether the grant holds, or `null` when the check failed and can't say. */
 export const kvSharedRegionAuthorized = async (
     actor: Actor,
     permission: string,
     deps: KvSharedRegionDeps,
-): Promise<boolean> => {
+): Promise<boolean | null> => {
     if (!deps.enabled) return false;
     try {
         return await deps.checkPermission(actor, permission);
     } catch {
-        return false;
+        return null;
     }
 };
 

src/backend/services/events/consentSettles.test.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import {
+    CONSENT_CONFIRMED_TTL_MS,
+    CONSENT_SETTLE_WINDOW_MS,
+    CONSENT_SETTLES_PER_WINDOW,
+    ConsentSettles,
+} from './consentSettles.js';
+
+beforeEach(() => {
+    vi.useFakeTimers({ toFake: ['Date'] });
+});
+
+afterEach(() => {
+    vi.useRealTimers();
+});
+
+it('runs one check per pair at a time', () => {
+    const settles = new ConsentSettles();
+    expect(settles.begin('1|app')).toBe('start');
+    expect(settles.begin('1|app')).toBe('skip');
+    expect(settles.begin('2|app')).toBe('start');
+    settles.end('1|app');
+    expect(settles.begin('1|app')).toBe('start');
+});
+
+it('skips a pair a fresh read just confirmed, until that lapses', () => {
+    const settles = new ConsentSettles();
+    settles.begin('1|app');
+    settles.confirm('1|app');
+    settles.end('1|app');
+    expect(settles.begin('1|app')).toBe('skip');
+
+    vi.setSystemTime(Date.now() + CONSENT_CONFIRMED_TTL_MS + 1);
+    expect(settles.begin('1|app')).toBe('start');
+});
+
+it('stops settling once the window budget is spent, and resets after it', () => {
+    const settles = new ConsentSettles();
+    for (let i = 0; i < CONSENT_SETTLES_PER_WINDOW; i++)
+        expect(settles.take()).toBe(true);
+    expect(settles.take()).toBe(false);
+    // Spent means no check either: nothing it could settle.
+    expect(settles.begin('1|app')).toBe('spent');
+
+    vi.setSystemTime(Date.now() + CONSENT_SETTLE_WINDOW_MS);
+    expect(settles.begin('1|app')).toBe('start');
+    expect(settles.take()).toBe(true);
+});

src/backend/services/events/consentSettles.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Paces the settles a delivery starts when it finds background consent
+ * withdrawn. A settle ends rows for good, so a permission check that wrongly
+ * answers no must not end every row a delivery reaches: past a small budget per
+ * window the rest only defer. A pair a fresh read just found consented is not
+ * read again for a while, so a stale cached no costs one primary read, not one
+ * per delivery. Per process, and not a published limit.
+ */
+
+export const CONSENT_SETTLES_PER_WINDOW = 20;
+export const CONSENT_SETTLE_WINDOW_MS = 60_000;
+export const CONSENT_CONFIRMED_TTL_MS = 60_000;
+export const CONSENT_CONFIRMED_MAX_ENTRIES = 10_000;
+
+/** `spent`: the window's budget is gone, which is worth an alarm. */
+export type ConsentCheckStart = 'start' | 'skip' | 'spent';
+
+export class ConsentSettles {
+    readonly #inFlight = new Set<string>();
+    /** Insertion order is time order: re-confirming moves a key to the tail. */
+    readonly #confirmedAt = new Map<string, number>();
+    #windowStart = 0;
+    #taken = 0;
+
+    /** Whether to check `key` now; `start` must be paired with `end`. */
+    begin(key: string): ConsentCheckStart {
+        if (this.#inFlight.has(key)) return 'skip';
+        const confirmedAt = this.#confirmedAt.get(key);
+        if (confirmedAt !== undefined) {
+            if (Date.now() - confirmedAt <= CONSENT_CONFIRMED_TTL_MS)
+                return 'skip';
+            this.#confirmedAt.delete(key);
+        }
+        if (this.#left() === 0) return 'spent';
+        this.#inFlight.add(key);
+        return 'start';
+    }
+
+    end(key: string): void {
+        this.#inFlight.delete(key);
+    }
+
+    /** A fresh read found the consent standing. */
+    confirm(key: string): void {
+        this.#confirmedAt.delete(key);
+        this.#confirmedAt.set(key, Date.now());
+        this.#prune();
+    }
+
+    /** One settle from this window's budget; false once it is spent. */
+    take(): boolean {
+        if (this.#left() === 0) return false;
+        this.#taken++;
+        return true;
+    }
+
+    #left(): number {
+        if (Date.now() - this.#windowStart >= CONSENT_SETTLE_WINDOW_MS) {
+            this.#windowStart = Date.now();
+            this.#taken = 0;
+        }
+        return CONSENT_SETTLES_PER_WINDOW - this.#taken;
+    }
+
+    #prune(): void {
+        for (const [key, confirmedAt] of this.#confirmedAt) {
+            const overCap =
+                this.#confirmedAt.size > CONSENT_CONFIRMED_MAX_ENTRIES;
+            const headExpired =
+                Date.now() - confirmedAt > CONSENT_CONFIRMED_TTL_MS;
+            if (!overCap && !headExpired) break;
+            this.#confirmedAt.delete(key);
+        }
+    }
+}

src/backend/services/events/dispatch.integration.test.ts

@@ -80,7 +80,7 @@ const subscribeTo = async (subject: string) => {
 };
 
 describe('the write path reaches subscribers', () => {
-    it('omits values after the temporary KV fan-out threshold', async () => {
+    it('keeps values on the deliveries a capped KV fan-out makes', async () => {
         const key = `fanout-value-${Date.now()}`;
         const socketIds = Array.from({ length: 129 }, (_, i) =>
             `kv-fanout-${i}`,
@@ -117,9 +117,15 @@ describe('the write path reaches subscribers', () => {
                     envelope.event.op === 'set' &&
                     (envelope.event as { key?: string }).key === key,
             );
-            expect(sent.every((envelope) => !('value' in envelope.event))).toBe(
-                true,
-            );
+            // The value rides on the first 128 deliveries, which is all of
+            // them at this cap.
+            expect(
+                sent.every(
+                    (envelope) =>
+                        (envelope.event as { value?: unknown }).value ===
+                        're-read',
+                ),
+            ).toBe(true);
         } finally {
             await Promise.all(
                 socketIds.map((socketId) => events().reapSocket(userId, socketId)),

src/backend/services/events/durable.integration.test.ts

@@ -18,8 +18,9 @@
  */
 
 /**
- * Durable subscriptions end to end: the routes that create and revoke them, who
- * each credential shape may see, and what a write costs once one exists.
+ * Durable subscriptions end to end: the routes that create, update and revoke
+ * them, who each credential shape may see, and what a write costs once one
+ * exists.
  *
  * The cost assertions are the point of the whole cache. A subscription made
  * here is deliverable here without the table, and a region that has never seen
@@ -36,6 +37,7 @@ import {
 } from '../../controllers/events/limits.js';
 import { DEFAULT_FREE_SUBSCRIPTION } from '../metering/consts.js';
 import { makeActor } from '../../core/actor.js';
+import { DURABLE_CONTEXT_MAX_BYTES } from '../../stores/events/DurableSubscriptionStore.js';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
 import type { IConfig } from '../../types.js';
 import { appSocketRoom } from '../socket/SocketService.js';
@@ -97,6 +99,13 @@ const listSubscriptions = (token: string, query = ''): Promise<ApiResponse> =>
 const unsubscribe = (token: string, subId: string): Promise<ApiResponse> =>
     call('POST', '/events/unsubscribe', token, { subId });
 
+const updateContext = (
+    token: string,
+    subId: string,
+    body: object,
+): Promise<ApiResponse> =>
+    call('POST', '/events/subscriptions/update', token, { subId, ...body });
+
 const subIdsOf = (response: ApiResponse): string[] =>
     (response.body.items as Array<{ subId: string }>).map((row) => row.subId);
 
@@ -588,6 +597,140 @@ describe('what each credential sees and removes', () => {
     });
 });
 
+describe('replacing a durable subscription`s context', () => {
+    const resetBudget = () =>
+        env.server.clients.redis.del(
+            `rate:${EVENTS_SUBSCRIBE_LIMIT.scope}:${userId}`,
+        );
+
+    /** The row this region would deliver against, read from its cache. */
+    const cachedRow = async (subId: string) => {
+        const stored =
+            await env.server.stores.durableSubscription.getBySubId(subId);
+        const cached = await env.server.stores.eventSubscription.getForTokens(
+            userId,
+            [stored!.token],
+        );
+        return cached.find((row) => row.subId === subId);
+    };
+
+    it('keeps the id and the backlog, and delivers the new value from then on', async () => {
+        await resetBudget();
+        await clearRows();
+        const created = await subscribe(appOneToken, {
+            context: { label: 'old' },
+        });
+        const subId = created.body.subId as string;
+        await env.server.stores.pendingDelivery.enqueue(subId, {
+            id: uuidv4(),
+            subject: `fs:${anchor}`,
+            op: 'gap',
+            reason: 'backlog_overflow',
+            ts: Date.now(),
+        });
+
+        const updated = await updateContext(appOneToken, subId, {
+            context: { label: 'new', region: 'eu' },
+        });
+
+        expect(updated.status).toBe(200);
+        expect(updated.body).toMatchObject({
+            subId,
+            contextKeys: ['label', 'region'],
+        });
+        expect(updated.body.contextHash).not.toBe(created.body.contextHash);
+        expect(subIdsOf(await listSubscriptions(appOneToken))).toEqual([
+            subId,
+        ]);
+        await expect(
+            env.server.stores.pendingDelivery.depth(subId),
+        ).resolves.toBe(1);
+        expect((await cachedRow(subId))?.context).toBe(
+            JSON.stringify({ label: 'new', region: 'eu' }),
+        );
+    });
+
+    it('clears it with null', async () => {
+        await resetBudget();
+        await clearRows();
+        const subId = (
+            await subscribe(env.users.user.token, { context: { a: 1 } })
+        ).body.subId as string;
+
+        const cleared = await updateContext(env.users.user.token, subId, {
+            context: null,
+        });
+
+        expect(cleared.status).toBe(200);
+        expect(cleared.body).toMatchObject({
+            contextKeys: null,
+            contextHash: null,
+        });
+        expect((await cachedRow(subId))?.context).toBeNull();
+    });
+
+    it('refuses a body without context, and a context over the cap', async () => {
+        await resetBudget();
+        await clearRows();
+        const subId = (
+            await subscribe(env.users.user.token, { context: { a: 1 } })
+        ).body.subId as string;
+
+        const missing = await updateContext(env.users.user.token, subId, {});
+        expect(missing.status).toBe(400);
+        expect(missing.body.code).toBe('bad_request');
+
+        const tooLarge = await updateContext(env.users.user.token, subId, {
+            context: { blob: 'x'.repeat(DURABLE_CONTEXT_MAX_BYTES) },
+        });
+        expect(tooLarge.status).toBe(413);
+        expect(tooLarge.body.code).toBe('events_context_too_large');
+        expect((await cachedRow(subId))?.context).toBe(
+            JSON.stringify({ a: 1 }),
+        );
+    });
+
+    it('answers a row the caller does not hold as absent', async () => {
+        await resetBudget();
+        await clearRows();
+        const mine = (await subscribe(appOneToken)).body.subId as string;
+
+        for (const token of [appTwoToken, appOneAccessToken, readUrlToken]) {
+            const refused = await updateContext(token, mine, {
+                context: { stolen: true },
+            });
+            expect(refused.status).toBe(404);
+            expect(refused.body.code).toBe('subscription_does_not_exist');
+        }
+        expect((await cachedRow(mine))?.context).toBeNull();
+    });
+
+    it('spends the subscribe budget', async () => {
+        await resetBudget();
+        try {
+            await clearRows();
+            const mine = (await subscribe(env.users.user.token)).body
+                .subId as string;
+            for (let i = 1; i < EVENTS_SUBSCRIBE_LIMIT.limit; i++)
+                await updateContext(env.users.user.token, mine, {
+                    context: { i },
+                });
+
+            const refused = await subscribe(env.users.user.token);
+            expect(refused.status).toBe(429);
+            const refusedUpdate = await updateContext(
+                env.users.user.token,
+                mine,
+                { context: { i: -1 } },
+            );
+            expect(refusedUpdate.status).toBe(429);
+            expect(refusedUpdate.body.code).toBe('too_many_requests');
+        } finally {
+            await resetBudget();
+        }
+    });
+});
+
 describe('what a write costs once a durable row exists', () => {
     it('delivers straight after subscribe without reading the table', async () => {
         await clearRows();
@@ -729,6 +872,71 @@ describe('what a write costs once a durable row exists', () => {
 
         expect(deliveryOf(path)).toBeUndefined();
     });
+
+    it('publishes the expiry reap`s bump, so a region still caching the row drops it', async () => {
+        await clearRows();
+        const created = await subscribe(env.users.user.token, {
+            expiresAt: Math.floor(Date.now() / 1000) + 3600,
+        });
+        const subId = created.body.subId as string;
+        const stale =
+            await env.server.stores.durableSubscription.getBySubId(subId);
+        await env.server.clients.db.write(
+            `UPDATE \`${TABLE}\` SET \`expires_at\` = ? WHERE \`sub_id\` = ?`,
+            [Math.floor(Date.now() / 1000) - 1, subId],
+        );
+
+        const heard: Array<Record<string, unknown>> = [];
+        const listener = (_key: string, data: unknown) => {
+            heard.push(data as Record<string, unknown>);
+        };
+        const announced = vi.spyOn(
+            env.server.services.eventForward,
+            'announceGeneration',
+        );
+        env.server.clients.event.on(
+            'outer.pubsub.events.generationBumped',
+            listener as never,
+        );
+        try {
+            await expect(events().sweepExpired()).resolves.toBe(1);
+            // Over the addressed peer channel as well as the bus.
+            expect(announced).toHaveBeenCalledWith(
+                expect.objectContaining({ userId }),
+                true,
+                'subscription',
+            );
+        } finally {
+            env.server.clients.event.off(
+                'outer.pubsub.events.generationBumped',
+                listener as never,
+            );
+            announced.mockRestore();
+        }
+        const published = heard.find(
+            (bump) => bump.userId === userId && bump.durable === true,
+        );
+        expect(published).toBeDefined();
+
+        // Another region's Redis still holds the row; this one stands in for it.
+        await env.server.stores.eventSubscription.cacheDurable([stale!]);
+        delivered.length = 0;
+        const before = `${anchor}/stale-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: before });
+        await settle(before);
+        expect(deliveryOf(before)?.subId).toBe(subId);
+
+        await env.server.clients.event.emitAndWait(
+            'outer.pubsub.events.generationBumped',
+            published!,
+            { from_outside: true },
+        );
+        delivered.length = 0;
+        const after = `${anchor}/peer-swept-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: after });
+        await quiet();
+        expect(deliveryOf(after)).toBeUndefined();
+    });
 });
 
 describe('where a durable delivery is addressed', () => {
@@ -904,6 +1112,10 @@ describe('with events switched off', () => {
             }),
             offCall('GET', '/events/subscriptions'),
             offCall('POST', '/events/unsubscribe', { subId: 'user#nope' }),
+            offCall('POST', '/events/subscriptions/update', {
+                subId: 'user#nope',
+                context: null,
+            }),
         ]) {
             const response = await attempt;
             expect(response.status).toBe(503);

src/backend/services/events/fetch.integration.test.ts

@@ -28,6 +28,7 @@
 
 import { v4 as uuidv4 } from 'uuid';
 import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import { NotificationFeed } from '../../../puter-js/src/modules/notifications/lib/feed.js';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
 import type { IConfig } from '../../types.js';
 
@@ -449,6 +450,172 @@ describe('who a fetch may see', () => {
     });
 });
 
+describe('reading again from a saved cursor', () => {
+    const SUBJECT = 'notif:app-user';
+
+    /** One of the app's own messages, in a group of its own. */
+    const message = async (title: string, inApp = appUid): Promise<string> =>
+        (
+            await env.server.stores.notification.create({
+                userId,
+                value: { type: 'app.message', title },
+                type: 'app.message',
+                audience: 'app-user',
+                appUid: inApp,
+                group: `group-${uuidv4()}`,
+            })
+        ).uid as string;
+
+    /** A replacement pinned to `replacedAt`, as a group replace writes it. */
+    const replaceAt = (uid: string, title: string, replacedAt: number) =>
+        env.server.stores.notification.updateValue(
+            uid,
+            userId,
+            { type: 'app.message', title },
+            { onlyUnseen: true, replacedAt },
+        );
+
+    it('returns a notification replaced since, from an earlier page', async () => {
+        await clearMailboxes();
+        const uids: string[] = [];
+        for (let i = 0; i < 3; i++) uids.push(await message(`m${i}`));
+
+        const first = await fetchPage(appToken, { subject: SUBJECT, limit: 2 });
+        const last = await fetchPage(appToken, {
+            subject: SUBJECT,
+            limit: 2,
+            cursor: first.body.cursor,
+        });
+        expect(uidsOf(last)).toEqual([uids[2]]);
+        expect(last.body.cursor).toBeUndefined();
+
+        await env.server.services.notification.notifyUpdate(
+            uids[0],
+            userId,
+            { title: 'replaced' },
+            { type: 'app.message', replace: true },
+        );
+
+        const again = await fetchPage(appToken, {
+            subject: SUBJECT,
+            limit: 2,
+            cursor: first.body.cursor,
+        });
+        expect(uidsOf(again)).toEqual([uids[2], uids[0]]);
+        expect(again.body.items![1]).toMatchObject({
+            replaced: true,
+            notification: { title: 'replaced' },
+        });
+    });
+
+    it('pages through replacements without repeating or skipping one', async () => {
+        await clearMailboxes();
+        const uids: string[] = [];
+        for (let i = 0; i < 6; i++) uids.push(await message(`m${i}`));
+        const first = await fetchPage(appToken, { subject: SUBJECT, limit: 5 });
+        expect(first.body.cursor).toBeTruthy();
+
+        // One moment for all four, so only the row id orders them.
+        const at = Date.now();
+        for (const uid of uids.slice(0, 4)) await replaceAt(uid, 'again', at);
+
+        const seen: string[] = [];
+        let cursor = first.body.cursor;
+        do {
+            const page = await fetchPage(appToken, {
+                subject: SUBJECT,
+                limit: 2,
+                cursor,
+            });
+            seen.push(...uidsOf(page));
+            cursor = page.body.cursor;
+        } while (cursor);
+
+        expect(seen).toEqual([uids[5], ...uids.slice(0, 4)]);
+    });
+
+    it('leaves out a replacement in another app`s slice', async () => {
+        await clearMailboxes();
+        const other = await message('other app', otherAppUid);
+        const own = [await message('own 0'), await message('own 1')];
+        const first = await fetchPage(appToken, { subject: SUBJECT, limit: 1 });
+        expect(uidsOf(first)).toEqual([own[0]]);
+
+        await replaceAt(other, 'other app again', Date.now());
+
+        const again = await fetchPage(appToken, {
+            subject: SUBJECT,
+            cursor: first.body.cursor,
+        });
+        expect(uidsOf(again)).toEqual([own[1]]);
+    });
+
+    it('hands the SDK feed a replacement made while it was disconnected, once', async () => {
+        await clearMailboxes();
+        // One more than a page, so the feed holds a cursor.
+        const uids: string[] = [];
+        for (let i = 0; i < 201; i++) uids.push(await message(`message ${i}`));
+
+        const appActor = (await env.server.services.auth.authenticate(appToken))
+            .actor!;
+        let reads = 0;
+        let receive: (delivery: { event: unknown }) => void = () => {};
+        const titles: string[] = [];
+        const feed = new NotificationFeed(
+            {
+                authToken: appToken,
+                events: {
+                    onLocal: async (
+                        _subject: string,
+                        handler: typeof receive,
+                    ) => {
+                        receive = handler;
+                        return { off: async () => {} };
+                    },
+                    fetch: async (options: Record<string, unknown>) => {
+                        const page =
+                            await env.server.services.events.fetchMissed(
+                                appActor,
+                                options,
+                            );
+                        reads++;
+                        return page;
+                    },
+                },
+            } as never,
+            (notification: { title: string | null }) => {
+                titles.push(String(notification.title));
+            },
+            {},
+        );
+        const gapRead = async () => {
+            const before = reads;
+            receive({ event: { op: 'gap', reason: 'reconnect' } });
+            await vi.waitFor(() => expect(reads).toBe(before + 1));
+            await new Promise((resolve) => setTimeout(resolve, 0));
+        };
+
+        try {
+            await feed.start();
+            expect(titles).toHaveLength(201);
+
+            await env.server.services.notification.notifyUpdate(
+                uids[0],
+                userId,
+                { title: 'replaced while away' },
+                { type: 'app.message', replace: true },
+            );
+            await gapRead();
+            await gapRead();
+        } finally {
+            await feed.off();
+        }
+
+        expect(titles).toHaveLength(202);
+        expect(titles.at(-1)).toBe('replaced while away');
+    });
+});
+
 describe('cross-user isolation', () => {
     it('returns nothing of another account, whatever the subject names', async () => {
         await clearMailboxes();

src/backend/services/events/forwarding.test.ts

@@ -79,6 +79,14 @@ vi.mock(import('@opentelemetry/api'), async (importOriginal) => ({
                     metricCalls.push({ name, value, attributes });
                 },
             }),
+            createHistogram: (name: string) => ({
+                record: (
+                    value: number,
+                    attributes: Record<string, unknown> = {},
+                ) => {
+                    metricCalls.push({ name, value, attributes });
+                },
+            }),
         }),
     },
 }));
@@ -441,6 +449,7 @@ const makeRegion = (
             ): Promise<{
                 row: DurableSubscription;
                 bump: { userId: number; generation: number };
+                reaped: [];
             }> => {
                 const row = durableRow({
                     holderUserId: input.holderUserId as number,
@@ -469,6 +478,7 @@ const makeRegion = (
                             row.ownerUserId,
                         ),
                     },
+                    reaped: [],
                 };
             },
         },
@@ -507,7 +517,7 @@ const makeRegion = (
     region.events.worker = {
         invoke: async (invocation: WorkerInvocation) => {
             region.invoked.push(invocation);
-            return workerOutcome;
+            return { outcome: workerOutcome };
         },
     };
     region.forward.onServerStart();
@@ -1361,7 +1371,8 @@ describe('a delivery owed to exactly one consumer', () => {
         await dispatch(west);
         await arrived(west);
 
-        expect(west.posts).toEqual([]);
+        // Deliveries only: its own connect announces presence to peers.
+        expect(deliveriesIn(west)).toEqual([]);
         expect(west.sent[0]).toMatchObject({ ackRequired: true });
     });
 

src/backend/services/events/handlers.integration.test.ts

@@ -39,6 +39,7 @@ import { v4 as uuidv4 } from 'uuid';
 import {
     EVENTS_COALESCE_WINDOW_MS,
     EVENTS_HANDLER_PUBLISH_BATCH,
+    EVENTS_HANDLER_PUBLISH_LIMIT,
     EVENTS_HANDLER_SOURCE_MAX_BYTES,
     EVENTS_SUSPENDED_PENDING_CAP,
     EVENTS_WORKER_SOURCE_MAX_BYTES,
@@ -219,7 +220,8 @@ afterAll(async () => {
 
 beforeEach(async () => {
     await env.server.clients.db.write('DELETE FROM `event_handlers`', []);
-    for (const row of await durable().listActiveForHolder(userId, null))
+    for (const row of (await durable().listActiveForHolderPage(userId, null, 0))
+        .rows)
         await durable().remove(row);
     await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []);
     await env.server.stores.eventSubscription.rebuildDurable(userId, []);
@@ -341,6 +343,61 @@ describe('who may publish a handler', () => {
     });
 });
 
+describe('what a handler may refer to', () => {
+    // The file publishes close to the per-minute budget without these.
+    beforeEach(async () => {
+        await env.server.clients.redis.del(
+            `rate:${EVENTS_HANDLER_PUBLISH_LIMIT.scope}:${userId}`,
+        );
+    });
+
+    it('refuses a handler that names a variable it does not bind', async () => {
+        const refused = await publish(appToken, {
+            name: 'ingestUpload',
+            source: 'async ({ event }) => { await send(endpoint, event.path); }',
+        });
+
+        expect(refused.status).toBe(400);
+        expect(refused.body.code).toBe('events_handler_free_variable');
+        expect(refused.body.message).toContain('`send`');
+        const listed = await call('GET', '/events/handlers/list', appToken);
+        expect(listed.body.handlers).toEqual([]);
+    });
+
+    it('refuses the ambient SDK, which a handler never has', async () => {
+        const refused = await publish(appToken, {
+            name: 'ingestUpload',
+            source: 'async ({ event }) => { await puter.fs.stat(event.path); }',
+        });
+
+        expect(refused.status).toBe(400);
+        expect(refused.body.code).toBe('events_handler_free_variable');
+    });
+
+    it('stops a set at the item that does', async () => {
+        const refused = await call(
+            'POST',
+            '/events/handlers/publishAll',
+            appToken,
+            {
+                handlers: [
+                    { name: 'indexDocument', source: NEXT_SOURCE },
+                    { name: 'ingestUpload', source: 'async () => leak()' },
+                ],
+            },
+        );
+
+        expect(refused.status).toBe(400);
+        expect(refused.body.code).toBe('events_handler_free_variable');
+        const listed = await call('GET', '/events/handlers/list', appToken);
+        expect(
+            (listed.body.handlers as Array<{ name: string }>).map(
+                (h) => h.name,
+            ),
+        ).toEqual(['indexDocument']);
+    });
+});
+
 describe('publishing a set', () => {
     it('takes a build step`s handlers in one call', async () => {
         const published = await call(
@@ -624,8 +681,14 @@ describe('the total-source cap', () => {
         { timeout: 30_000 },
         async () => {
             // Every handler at the per-handler max: the fewest handlers that
-            // can reach the total cap.
-            const bigSource = 'x'.repeat(EVENTS_HANDLER_SOURCE_MAX_BYTES);
+            // can reach the total cap. Padded in a comment so it still passes
+            // the free-variable scan.
+            const prefix = 'async () => {} /*';
+            const padded = (fill: string) =>
+                `${prefix}${fill.repeat(
+                    EVENTS_HANDLER_SOURCE_MAX_BYTES - prefix.length - 2,
+                )}*/`;
+            const bigSource = padded('x');
             const names = Array.from(
                 {
                     length:
@@ -666,7 +729,7 @@ describe('the total-source cap', () => {
             // same size, different content, so it must not double-count.
             const replace = await publish(appToken, {
                 name: 'big0',
-                source: 'y'.repeat(EVENTS_HANDLER_SOURCE_MAX_BYTES),
+                source: padded('y'),
                 replace: true,
             });
             expect(replace.status).toBe(200);
@@ -715,7 +778,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(404);
         expect(refused.body.code).toBe('events_handler_not_found');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('binds an inline body whose hash is what is published', async () => {
@@ -743,7 +808,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(409);
         expect(refused.body.code).toBe('events_handler_hash_mismatch');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('still needs a name for a subscription owed to one consumer', async () => {

src/backend/services/events/kvHandleRevoke.integration.test.ts

@@ -28,8 +28,21 @@
  */
 
 import { v4 as uuidv4 } from 'uuid';
-import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
-import { EVENTS_COALESCE_WINDOW_MS } from '../../controllers/events/limits.js';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import {
+    EVENTS_COALESCE_WINDOW_MS,
+    EVENTS_KV_HANDLE_LIMIT,
+    EVENTS_SUBSCRIBE_LIMIT,
+} from '../../controllers/events/limits.js';
 import { makeActor, type Actor } from '../../core/actor.js';
 import { runWithContext } from '../../core/context.js';
 import {
@@ -38,8 +51,12 @@ import {
     type PuterTestEnv,
 } from '../../testUtil.js';
 import type { IConfig } from '../../types.js';
+import { SESSION_SUBSCRIPTION_TTL_SECONDS } from '../../stores/events/EventSubscriptionStore.js';
+import { kvShareGrantAtEveryLevel } from '../kvShare/kvShares.js';
+import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import type { DeliveryEnvelope } from './EventsService.js';
 import { kvAnchorToken } from './subjects.js';
+import { RecordingWorkerInvoker } from './workerSeam.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 const SOCKET_ID = 'kv-revoke-socket';
@@ -145,6 +162,7 @@ const clearRows = async () => {
 beforeAll(async () => {
     env = await setupPuterTestEnv({
         events: { enabled: true, kvHandles: true },
+        kv: { shareHandles: true },
         // Seeded accounts carry no email, which the plan machinery reads as a
         // temporary account — and a temporary account holds no durable rows.
         unlimitedMetering: true,
@@ -221,13 +239,151 @@ describe('revoking a handle', () => {
         expect(delivered).toEqual([]);
     });
 
+    describe('ending session subscriptions made through it', () => {
+        let send: ReturnType<typeof vi.spyOn>;
+        beforeEach(() => {
+            send = vi.spyOn(env.server.services.socket, 'send');
+        });
+        afterEach(() => {
+            send.mockRestore();
+        });
+
+        const endedOn = (socketId: string, subId: string) =>
+            vi.waitFor(
+                () =>
+                    expect(send).toHaveBeenCalledWith(
+                        { socket: socketId },
+                        'events.ended',
+                        expect.objectContaining({
+                            subId,
+                            code: 'subscription_ended',
+                            reason: 'permission_revoked',
+                        }),
+                    ),
+                { timeout: 5_000, interval: 25 },
+            );
+
+        const heldOn = async (userId: number, socketId: string) =>
+            (
+                await env.server.stores.eventSubscription.listForSocket(
+                    userId,
+                    socketId,
+                )
+            ).map((row) => row.subId);
+
+        it('ends them when the owner revokes it, and tells the connection', async () => {
+            await clearRows();
+            const { handle } = await mint();
+            const { sub } = await events().subscribe(guest.actor, SOCKET_ID, {
+                subject: `kv:${handle}:*`,
+            });
+
+            await shares().revokeKvHandle(owner.actor, handle);
+
+            await endedOn(SOCKET_ID, sub.subId);
+            expect(await heldOn(guest.id, SOCKET_ID)).toEqual([]);
+            expect(await ownerWatches()).toBe(false);
+        });
+
+        it('still suspends durable subscriptions when ending them fails', async () => {
+            await clearRows();
+            const { handle } = await mint();
+            const durable = await subscribeDurable(handle);
+            await events().subscribe(guest.actor, SOCKET_ID, {
+                subject: `kv:${handle}:*`,
+            });
+            // Retired with the grant left alone, so only the retire settle
+            // can suspend the row.
+            await env.server.stores.kvShareHandle.retireMany([handle], 'owner');
+
+            const store = env.server.stores.eventSubscription;
+            const listing = vi
+                .spyOn(store, 'listForKvHandles')
+                .mockRejectedValue(new Error('redis unavailable'));
+            try {
+                await events().settleRetiredKvHandles({
+                    handles: [handle],
+                    userId: guest.id,
+                });
+                await suspendedRow(durable.subId);
+            } finally {
+                listing.mockRestore();
+            }
+        });
+
+        it('ends them when the holder gives it up', async () => {
+            await clearRows();
+            const { handle } = await mint();
+            const { sub } = await events().subscribe(guest.actor, SOCKET_ID, {
+                subject: `kv:${handle}:*`,
+            });
+
+            await shares().removeKvShare(guest.actor, handle);
+
+            await endedOn(SOCKET_ID, sub.subId);
+            expect(await heldOn(guest.id, SOCKET_ID)).toEqual([]);
+        });
+
+        it('ends them on the connection`s next refresh when the handle stopped standing out of sight', async () => {
+            await clearRows();
+            const socketId = 'kv-revoke-refresh';
+            const retired = await mint();
+            const withdrawn = await mint('workspace:withdrawn:');
+            const kept = await mint('workspace:kept:');
+
+            vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
+            try {
+                const subs = [];
+                for (const { handle } of [retired, withdrawn, kept])
+                    subs.push(
+                        (
+                            await events().subscribe(guest.actor, socketId, {
+                                subject: `kv:${handle}:*`,
+                            })
+                        ).sub,
+                    );
+
+                // Neither goes through a retire this region hears about: a
+                // handle retired from another region, and a grant withdrawn
+                // with its handle left live.
+                await env.server.stores.kvShareHandle.retireMany(
+                    [retired.handle],
+                    'owner',
+                );
+                const grant = await env.server.stores.kvShareHandle.getByHandle(
+                    withdrawn.handle,
+                );
+                await env.server.services.permission.revokeUserUserPermissionSubtree(
+                    owner.actor,
+                    guest.username,
+                    kvShareGrantAtEveryLevel(grant!.permission),
+                    { reason: 'test' },
+                );
+                expect(await heldOn(guest.id, socketId)).toHaveLength(3);
+
+                await vi.advanceTimersByTimeAsync(
+                    (SESSION_SUBSCRIPTION_TTL_SECONDS * 1000) / 3,
+                );
+
+                await endedOn(socketId, subs[0].subId);
+                await endedOn(socketId, subs[1].subId);
+                expect(await heldOn(guest.id, socketId)).toEqual([
+                    subs[2].subId,
+                ]);
+            } finally {
+                await events().reapSocket(guest.id, socketId);
+                vi.useRealTimers();
+            }
+        });
+    });
+
     it('finds its subscriptions on the holder index, not by scanning', async () => {
         await clearRows();
         const { handle } = await mint();
         const sub = await subscribeDurable(handle);
 
         const store = env.server.stores.durableSubscription;
-        const byHolder = vi.spyOn(store, 'listActiveForHolder');
+        const byHolder = vi.spyOn(store, 'listActiveForHolderPage');
         const settle = vi.spyOn(events(), 'settleRevokedGrant');
 
         try {
@@ -244,7 +400,9 @@ describe('revoking a handle', () => {
             for (const call of byHolder.mock.calls)
                 expect(call.slice(1)).toEqual([
                     null,
-                    { includeSuspended: true },
+                    0,
+                    undefined,
+                    { includeSuspended: true, excludeSchedules: true },
                 ]);
         } finally {
             byHolder.mockRestore();
@@ -286,7 +444,9 @@ describe('revoking a handle', () => {
         const page = await shares().listKvHandles(owner.actor, {
             limit: 100,
         });
-        const deeperRow = page.items.find((one) => one.handle === deeper.handle);
+        const deeperRow = page.items.find(
+            (one) => one.handle === deeper.handle,
+        );
         expect(deeperRow?.revokedAt).toBeTypeOf('number');
         expect((await countLive()).total).toBe(before.total);
     });
@@ -412,6 +572,307 @@ describe('revoking a handle', () => {
     });
 });
 
+describe('a queued row whose share can`t be checked right now', () => {
+    const handles = () => env.server.stores.kvShareHandle;
+    const pending = () => env.server.stores.pendingDelivery;
+
+    /**
+     * A durable `single` on a fresh handle, with a backlog waiting and the
+     * lease its first hand-over took lapsed, so a sweep reads it again.
+     */
+    const queued = async () => {
+        await clearRows();
+        const { handle } = await mint();
+        const sub = await subscribeDurable(handle);
+        await ownerWrites(`${PREFIX}messages:1`, { body: 'queued' });
+        const lease = `ev:ql:{${sub.subId}}`;
+        await vi.waitFor(
+            async () => {
+                expect(await pending().depth(sub.subId)).toBeGreaterThan(0);
+                expect(await env.server.clients.redis.zcard(lease)).toBe(1);
+            },
+            { timeout: 5_000, interval: 25 },
+        );
+        await env.server.clients.redis.del(lease);
+        return { handle, sub, depth: await pending().depth(sub.subId) };
+    };
+
+    /** One sweep pass, reporting whether it went for this row's backlog. */
+    const sweepClaims = async (subId: string) => {
+        const claim = vi.spyOn(pending(), 'claim');
+        try {
+            await events().sweepPending();
+            return claim.mock.calls.some(([id]) => id === subId);
+        } finally {
+            claim.mockRestore();
+        }
+    };
+
+    it('keeps the row and its backlog when the share lookup fails', async () => {
+        const { sub, depth } = await queued();
+
+        const failing = vi
+            .spyOn(handles(), 'getLive')
+            .mockRejectedValue(new Error('lookup unavailable'));
+        try {
+            expect(await sweepClaims(sub.subId)).toBe(false);
+        } finally {
+            failing.mockRestore();
+        }
+
+        expect((await rowOf(sub.subId))?.suspended_reason).toBeNull();
+        expect(await pending().depth(sub.subId)).toBe(depth);
+        expect(await sweepClaims(sub.subId)).toBe(true);
+    });
+
+    it('holds the backlog while the owner is suspended, and hands it out after', async () => {
+        const { handle, sub, depth } = await queued();
+        const live = await handles().getLive(handle);
+
+        const suspended = vi
+            .spyOn(handles(), 'getLive')
+            .mockResolvedValue({ ...live!, ownerSuspended: true });
+        try {
+            expect(await sweepClaims(sub.subId)).toBe(false);
+        } finally {
+            suspended.mockRestore();
+        }
+        expect((await rowOf(sub.subId))?.suspended_reason).toBeNull();
+        expect(await pending().depth(sub.subId)).toBe(depth);
+
+        expect(await sweepClaims(sub.subId)).toBe(true);
+        expect((await rowOf(sub.subId))?.suspended_reason).toBeNull();
+    });
+
+    it('still settles a share that is gone', async () => {
+        const { sub } = await queued();
+
+        const gone = vi.spyOn(handles(), 'getLive').mockResolvedValue(null);
+        try {
+            expect(await sweepClaims(sub.subId)).toBe(false);
+        } finally {
+            gone.mockRestore();
+        }
+
+        expect((await rowOf(sub.subId))?.suspended_reason).toBe(
+            'permission_revoked',
+        );
+        expect(await pending().depth(sub.subId)).toBe(0);
+    });
+});
+
+describe('a queued backlog after its share drops to watch', () => {
+    const HANDLER = 'onChange';
+    const pending = () => env.server.stores.pendingDelivery;
+    const socket = () => env.server.services.socket;
+    const soon = (check: () => unknown) =>
+        vi.waitFor(check, { timeout: 5_000, interval: 25 });
+
+    let app: { uid: string; id: number };
+    let guestApp: Actor;
+    let worker: RecordingWorkerInvoker;
+    let previousWorker: ReturnType<typeof events>['worker'];
+
+    beforeAll(async () => {
+        const name = `kv-queued-${uuidv4().slice(0, 8)}`;
+        app = await env.server.stores.app.create(
+            {
+                name,
+                title: 'Queued Values',
+                index_url: `https://${name}.example.test/index.html`,
+            },
+            { ownerUserId: owner.id },
+        );
+        guestApp = makeActor({
+            user: guest.actor.user as never,
+            app: { uid: app.uid, id: app.id },
+        });
+        // The holder launching the app, and letting its handler run unattended.
+        for (const permission of [
+            'flag:app-is-authenticated',
+            EVENTS_BACKGROUND_PERMISSION,
+        ])
+            await env.server.services.permission.grantUserAppPermission(
+                guest.actor,
+                app.uid,
+                permission,
+            );
+        await events().publishHandler(owner.actor, {
+            appUid: app.uid,
+            name: HANDLER,
+            source: 'async ({ event }) => { console.log(event.key); }',
+        });
+    });
+
+    beforeEach(async () => {
+        await clearRows();
+        const limits = [EVENTS_KV_HANDLE_LIMIT, EVENTS_SUBSCRIBE_LIMIT];
+        for (const user of [owner, guest])
+            for (const limit of limits)
+                await env.server.clients.redis.del(
+                    `rate:${limit.scope}:${user.id}`,
+                );
+        previousWorker = events().worker;
+        worker = new RecordingWorkerInvoker();
+        events().worker = worker;
+    });
+
+    afterEach(() => {
+        events().worker = previousWorker;
+    });
+
+    /** Sharing a region again is how its level changes. */
+    const shareAt = (
+        prefix: string,
+        access: 'read' | 'watch',
+        appUid?: string,
+    ) =>
+        shares().mintKvHandle(
+            owner.actor,
+            {
+                prefix,
+                recipient: { username: guest.username },
+                access,
+                ...(appUid ? { appUid } : {}),
+            },
+            { defaultAccess: 'watch' },
+        );
+
+    const subscribeValues = async (handle: string, actor = guest.actor) =>
+        (
+            await events().subscribeDurable(actor, {
+                subject: `kv:${handle}:*`,
+                delivery: 'single',
+                handlerName: HANDLER,
+                includeValue: true,
+            })
+        ).sub;
+
+    const writes = (key: string, value: unknown, appUid?: string) =>
+        runWithContext({ actor: owner.actor }, () =>
+            env.server.drivers.kvStore.set({
+                key,
+                value,
+                ...(appUid ? { optConfig: { appUuid: appUid } } : {}),
+            }),
+        );
+
+    /** One entry queued and leased to nobody, so the next drain claims it. */
+    const backlogged = async (subId: string) => {
+        await soon(async () => expect(await pending().depth(subId)).toBe(1));
+        await env.server.clients.redis.del(`ev:ql:{${subId}}`);
+    };
+
+    const deliveredTo = (subId: string) =>
+        delivered.find((one) => one.subId === subId);
+
+    const ranFor = (subId: string) =>
+        worker.recorded
+            .filter((one) => one.subId === subId)
+            .map((one) => one.event);
+
+    /** Run `step` with the holder connected in this region. */
+    const connected = async (step: () => Promise<unknown>) => {
+        const has = vi.spyOn(socket(), 'has').mockReturnValue(true);
+        try {
+            await step();
+        } finally {
+            has.mockRestore();
+        }
+    };
+
+    it('hands a connected client the backlog without the value', async () => {
+        const prefix = 'queued:socket:';
+        const { handle } = await shareAt(prefix, 'read');
+        const sub = await subscribeValues(handle);
+        await writes(`${prefix}secret`, 'private-value');
+        await backlogged(sub.subId);
+
+        await shareAt(prefix, 'watch');
+        delivered.length = 0;
+        await connected(() => events().sweepPending());
+
+        const event = deliveredTo(sub.subId)?.event;
+        expect(event).toMatchObject({ key: 'secret', op: 'set' });
+        expect(event).not.toHaveProperty('value');
+    });
+
+    it('leaves the value off an older entry a new write hands out', async () => {
+        const prefix = 'queued:next-write:';
+        const { handle } = await shareAt(prefix, 'read');
+        const sub = await subscribeValues(handle);
+        await writes(`${prefix}first`, 'private-value');
+        await backlogged(sub.subId);
+
+        await shareAt(prefix, 'watch');
+        delivered.length = 0;
+        await connected(async () => {
+            await writes(`${prefix}second`, 'later');
+            await soon(() => expect(deliveredTo(sub.subId)).toBeDefined());
+        });
+
+        const event = deliveredTo(sub.subId)?.event;
+        expect(event).toMatchObject({ key: 'first' });
+        expect(event).not.toHaveProperty('value');
+    });
+
+    it('hands a handler run the backlog without the value', async () => {
+        const prefix = 'queued:worker:';
+        const { handle } = await shareAt(prefix, 'read', app.uid);
+        const sub = await subscribeValues(handle, guestApp);
+        await writes(`${prefix}secret`, 'private-value', app.uid);
+        await soon(() => expect(ranFor(sub.subId)).toHaveLength(1));
+        expect(ranFor(sub.subId)[0]).toMatchObject({ value: 'private-value' });
+
+        await shareAt(prefix, 'watch', app.uid);
+        await env.server.clients.redis.del(`ev:ql:{${sub.subId}}`);
+        await events().sweepPending();
+
+        const [, again] = ranFor(sub.subId);
+        expect(again).toMatchObject({ key: 'secret' });
+        expect(again).not.toHaveProperty('value');
+    });
+
+    it('hands a delivery its client declined to the handler without the value', async () => {
+        const prefix = 'queued:declined:';
+        const { handle } = await shareAt(prefix, 'read', app.uid);
+        const sub = await subscribeValues(handle, guestApp);
+        delivered.length = 0;
+        await connected(async () => {
+            await writes(`${prefix}secret`, 'private-value', app.uid);
+            await soon(() => expect(deliveredTo(sub.subId)).toBeDefined());
+        });
+        const envelope = deliveredTo(sub.subId)!;
+        expect(envelope.event).toMatchObject({ value: 'private-value' });
+
+        await shareAt(prefix, 'watch', app.uid);
+        await events().declineDelivery(guestApp, {
+            subId: sub.subId,
+            id: envelope.ackId,
+        });
+
+        const [run] = ranFor(sub.subId);
+        expect(run).toMatchObject({ key: 'secret' });
+        expect(run).not.toHaveProperty('value');
+    });
+
+    it('leaves the owner`s own backlog its values', async () => {
+        const prefix = 'queued:owner:';
+        const { handle } = await shareAt(prefix, 'watch');
+        const sub = await subscribeValues(handle, owner.actor);
+        await writes(`${prefix}mine`, 'kept');
+        await backlogged(sub.subId);
+
+        delivered.length = 0;
+        await connected(() => events().sweepPending());
+
+        expect(deliveredTo(sub.subId)?.event).toMatchObject({
+            key: 'mine',
+            value: 'kept',
+        });
+    });
+});
+
 describe('the owner`s handle listing', () => {
     it('shows what was shared, retired handles included', async () => {
         const { handle } = await mint();

src/backend/services/events/kvHandles.integration.test.ts

@@ -49,12 +49,15 @@ import { DEFAULT_FREE_SUBSCRIPTION } from '../metering/consts.js';
 import type { DeliveryEnvelope } from './EventsService.js';
 import { kvSharePermission } from '../kvShare/kvShares.js';
 import { seedKvShareHandles } from '../kvShare/kvShareTestUtil.js';
+import { MATCH_PATTERN_MAX_LENGTH } from './matcher.js';
 import { isKvHandleId, kvAnchorToken } from './subjects.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 const SOCKET_ID = 'kv-handle-socket';
 const TABLE = 'event_subscriptions';
 const PREFIX = 'workspace:abc:';
+/** Long enough that anything composed onto it passes the indexed-prefix cap. */
+const LONG_PREFIX = `workspace:${'a'.repeat(150)}:`;
 
 interface TestUser {
     actor: Actor;
@@ -542,6 +545,58 @@ describe('subscribing through a handle', () => {
         }
     });
 
+    it('holds a pattern to its limit as the holder wrote it, whatever the prefix', async () => {
+        await clearRows();
+        const handles = [
+            (await mint()).handle,
+            (await mint({ prefix: LONG_PREFIX })).handle,
+        ];
+        const longest = `${'x'.repeat(MATCH_PATTERN_MAX_LENGTH - 1)}*`;
+        const over = `${'x'.repeat(MATCH_PATTERN_MAX_LENGTH)}*`;
+
+        for (const handle of handles) {
+            await expect(
+                events().subscribe(guest.actor, SOCKET_ID, {
+                    subject: `kv:${handle}:${longest}`,
+                }),
+            ).resolves.toBeDefined();
+            // The same refusal for both, naming only what the holder wrote.
+            await expect(
+                events().subscribe(guest.actor, SOCKET_ID, {
+                    subject: `kv:${handle}:${over}`,
+                }),
+            ).rejects.toMatchObject({
+                legacyCode: 'invalid_subject_pattern',
+                message: `Match pattern exceeds ${MATCH_PATTERN_MAX_LENGTH} characters: ${over}`,
+            });
+        }
+    });
+
+    it('describes its filter the same way whatever the prefix', async () => {
+        const handles = [
+            (await mint()).handle,
+            (await mint({ prefix: LONG_PREFIX })).handle,
+        ];
+        for (const pattern of ['messages:*', 'messages*', 'a:b:c:d:e:f']) {
+            await clearRows();
+            const views = [];
+            for (const handle of handles) {
+                const { sub } = await events().subscribe(
+                    guest.actor,
+                    SOCKET_ID,
+                    { subject: `kv:${handle}:${pattern}` },
+                );
+                const [listed] = await events().listSubscriptions(
+                    guest.actor,
+                    SOCKET_ID,
+                );
+                views.push(sub.match, listed.match);
+                await events().reapSocket(guest.id, SOCKET_ID);
+            }
+            expect(new Set(views).size).toBe(1);
+        }
+    });
+
     it('refuses a user the handle was not granted to', async () => {
         await clearRows();
         const { handle } = await mint();
@@ -587,6 +642,24 @@ describe('a write in the shared region', () => {
         expect(JSON.stringify(delivered[0])).not.toContain(owner.uuid);
     });
 
+    it('reaches a pattern that runs past the limit once composed onto the prefix', async () => {
+        await clearRows();
+        const { handle } = await mint({ prefix: LONG_PREFIX });
+        const key = 'k'.repeat(200);
+        const { sub } = await events().subscribe(guest.actor, SOCKET_ID, {
+            subject: `kv:${handle}:${key}*`,
+        });
+        delivered.length = 0;
+
+        await ownerWrites(`${LONG_PREFIX}${key}:1`, 1);
+        await settled();
+
+        expect(delivered[0]).toMatchObject({
+            subId: sub.subId,
+            event: { key: `${key}:1` },
+        });
+    });
+
     it('reaches a durable subscription, key after key', async () => {
         await clearRows();
         const { handle } = await mint();
@@ -646,9 +719,10 @@ describe('a write in the shared region', () => {
                 ),
             { timeout: EVENTS_COALESCE_WINDOW_MS * 12, interval: 25 },
         );
-        expect(
-            delivered.find((one) => one.subId === sub.subId)?.event,
-        ).toMatchObject({ key: 'title', value: 'hello' });
+        // A `watch` share carries keys only, whatever the subscription asked.
+        const event = delivered.find((one) => one.subId === sub.subId)?.event;
+        expect(event).toMatchObject({ key: 'title' });
+        expect(event).not.toHaveProperty('value');
     });
 
     it('stays inside the region the handle was granted on', async () => {

src/backend/services/events/matcher.ts

@@ -46,6 +46,8 @@ export interface CompileMatchOptions {
     separator?: string | null;
     /** Let a wildcard-free pattern also match everything beneath it. */
     literalCoversSubtree?: boolean;
+    /** Longest pattern accepted; defaults to {@link MATCH_PATTERN_MAX_LENGTH}. */
+    maxLength?: number;
 }
 
 const WILDCARD = /[*?]/;
@@ -91,14 +93,12 @@ export function compileMatch(
     options: CompileMatchOptions = {},
 ): CompiledMatch {
     const separator = options.separator === undefined ? '/' : options.separator;
+    const maxLength = options.maxLength ?? MATCH_PATTERN_MAX_LENGTH;
 
     if (typeof pattern !== 'string' || pattern.length === 0)
         throw invalidPattern('is empty');
-    if (pattern.length > MATCH_PATTERN_MAX_LENGTH)
-        throw invalidPattern(
-            `exceeds ${MATCH_PATTERN_MAX_LENGTH} characters`,
-            pattern,
-        );
+    if (pattern.length > maxLength)
+        throw invalidPattern(`exceeds ${maxLength} characters`, pattern);
 
     const raw = separator === null ? [pattern] : pattern.split(separator);
     if (raw.length > MATCH_PATTERN_MAX_SEGMENTS)

src/backend/services/events/metering.test.ts

@@ -36,13 +36,17 @@ import {
     DurableQuotaRaceLost,
     type DurableSubscriptionInput,
 } from '../../stores/events/DurableSubscriptionStore.js';
-import type { DurableSubscription } from '../../stores/events/types.js';
+import {
+    scheduleToken,
+    type DurableSubscription,
+} from '../../stores/events/types.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
 import {
     DEFAULT_FREE_SUBSCRIPTION,
     DEFAULT_TEMP_SUBSCRIPTION,
 } from '../metering/consts.js';
 import type { UsageInput } from '../metering/types.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_COSTS } from './costs.js';
 import { EventsService } from './EventsService.js';
@@ -62,6 +66,12 @@ let solvent: Set<number>;
 /** Set to have the next create lose the last slot after its insert. */
 let raceLost: boolean;
 let emitted: Array<{ key: string; data: unknown }>;
+/** Rows one credit-sweep page takes; null for the service's own batch. */
+let pageSize: number | null;
+/** Rows one holder page takes; null for all of them in one page. */
+let holderPageSize: number | null;
+/** What `services.schedules` hears about changed schedule rows. */
+let schedulesChanged: (rows: readonly DurableSubscription[]) => void;
 
 const anchorUid = (): string => `docs-${seq}`;
 const anchorPath = (): string => `/u${userId}/Documents`;
@@ -99,7 +109,10 @@ const actorFor = (over: Partial<Actor> = {}): Actor =>
     }) as unknown as Actor;
 
 const appActor = (appUid: string): Actor =>
-    actorFor({ app: { uid: appUid, id: 1 }, effectiveApp: { uid: appUid, id: 1 } });
+    actorFor({
+        app: { uid: appUid, id: 1 },
+        effectiveApp: { uid: appUid, id: 1 },
+    });
 
 const anchorEntry = (): FSEntry =>
     ({
@@ -124,6 +137,9 @@ beforeEach(() => {
     solvent = new Set([userId]);
     raceLost = false;
     emitted = [];
+    pageSize = null;
+    holderPageSize = null;
+    schedulesChanged = () => undefined;
 
     service = new EventsService(
         { events: { enabled: true } } as IConfig,
@@ -155,26 +171,49 @@ beforeEach(() => {
                     return {
                         row,
                         bump: { userId: input.ownerUserId, generation: 1 },
+                        reaped: [],
                     };
                 },
-                listSuspendedPage: async (reason: string, afterId: number) =>
-                    afterId === 0
-                        ? {
-                              rows: rows.filter(
-                                  (row) => row.suspendedReason === reason,
-                              ),
-                              nextId: null,
-                          }
-                        : { rows: [], nextId: null },
-                listSuspendedForHolder: async (
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedPage: async (
+                    reason: string,
+                    afterId: number,
+                    batchSize: number,
+                ) => {
+                    const size = pageSize ?? batchSize;
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId && row.suspendedReason === reason,
+                        )
+                        .slice(0, size);
+                    return {
+                        rows: page.map(({ row }) => row),
+                        nextId: page.length < size ? null : page.at(-1)!.id,
+                    };
+                },
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedForHolderPage: async (
                     holderUserId: number,
                     reason: string,
-                ) =>
-                    rows.filter(
-                        (row) =>
-                            row.holderUserId === holderUserId &&
-                            row.suspendedReason === reason,
-                    ),
+                    afterId: number,
+                ) => {
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId &&
+                                row.holderUserId === holderUserId &&
+                                row.suspendedReason === reason,
+                        );
+                    const size = holderPageSize ?? page.length;
+                    const taken = page.slice(0, size);
+                    return {
+                        rows: taken.map(({ row }) => row),
+                        nextId: page.length > size ? taken.at(-1)!.id : null,
+                    };
+                },
                 resume: async (resuming: readonly DurableSubscription[]) => {
                     for (const row of resuming) {
                         resumed.push(row.subId);
@@ -233,10 +272,7 @@ beforeEach(() => {
             // plan allows, not about who agreed to it.
             permission: { check: async () => true },
             metering: {
-                bufferIncrementUsages: (
-                    actor: Actor,
-                    usages: UsageInput[],
-                ) => {
+                bufferIncrementUsages: (actor: Actor, usages: UsageInput[]) => {
                     for (const usage of usages)
                         metered.push({
                             userUuid: actor.user?.uuid,
@@ -247,6 +283,11 @@ beforeEach(() => {
                     solvent.has(Number(actor.user?.id ?? -1)),
                 getActorSubscription: async () => ({ id: plan }),
             },
+            schedules: {
+                onSubscriptionsChanged: async (
+                    changed: readonly DurableSubscription[],
+                ) => schedulesChanged(changed),
+            },
         } as never,
     );
 });
@@ -258,23 +299,105 @@ describe('a suspension waiting on a balance', () => {
             suspendedReason: 'no_credit',
         });
 
-    it('comes back once the account has budget again', async () => {
+    it('comes back once the account has budget again, and every region hears it', async () => {
         const row = suspended();
         rows.push(row);
 
-        await expect(service.sweepNoCredit()).resolves.toBe(1);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(1);
         expect(resumed).toEqual([row.subId]);
         expect(row.suspendedReason).toBeNull();
+        expect(emitted).toContainEqual({
+            key: 'outer.pubsub.events.generationBumped',
+            data: { userId, generation: 1, durable: true },
+        });
     });
 
     it('stays out of service while the account still has none', async () => {
         rows.push(suspended());
         solvent.clear();
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 
+    it('reaches rows past one run`s page cap across runs, then wraps', async () => {
+        // One row per page, so a run's 200 pages stop at 200 rows.
+        pageSize = 1;
+        for (let i = 0; i < 450; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 10_000 + i;
+        // Only the last holder has a balance again.
+        solvent = new Set([10_000 + 449]);
+
+        const first = testJobContext();
+        await expect(service.sweepNoCredit(first)).resolves.toBe(0);
+        expect(first.saved.at(-1)).toEqual({ afterId: 200 });
+
+        const second = testJobContext(first.saved.at(-1));
+        await expect(service.sweepNoCredit(second)).resolves.toBe(0);
+        expect(second.saved.at(-1)).toEqual({ afterId: 400 });
+
+        const third = testJobContext(second.saved.at(-1));
+        await expect(service.sweepNoCredit(third)).resolves.toBe(1);
+        expect(resumed).toEqual([rows[449].subId]);
+        // The end of the table wraps the cursor back to the start.
+        expect(third.saved.at(-1)).toBeNull();
+    });
+
+    it('stops between pages once its signal aborts, keeping the last full page', async () => {
+        pageSize = 1;
+        for (let i = 0; i < 5; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 20_000 + i;
+        const ctx = testJobContext();
+        const save = ctx.saveState;
+        ctx.saveState = async (next) => {
+            await save(next);
+            if (ctx.saved.length === 2) ctx.abort();
+        };
+
+        await service.sweepNoCredit(ctx);
+        expect(ctx.saved).toEqual([{ afterId: 1 }, { afterId: 2 }]);
+    });
+
+    it('resumes no later holder page once its signal aborts inside a page`s hook, keeping the checkpoint', async () => {
+        // A full sweep page, so finishing it would move the checkpoint.
+        pageSize = 3;
+        holderPageSize = 1;
+        for (let i = 0; i < 3; i++)
+            rows.push(
+                durableRow({
+                    subId: `sched-${i}`,
+                    token: scheduleToken(`schedule-${seq}-${i}`),
+                    suspendedAt: Math.floor(Date.now() / 1000),
+                    suspendedReason: 'no_credit',
+                }),
+            );
+        const ctx = testJobContext();
+        const heard: string[][] = [];
+        schedulesChanged = (changed) => {
+            heard.push(changed.map((row) => row.subId));
+            ctx.abort();
+        };
+
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(1);
+        expect(resumed).toEqual(['sched-0']);
+        expect(heard).toEqual([['sched-0']]);
+        expect(rows[1].suspendedReason).toBe('no_credit');
+        expect(ctx.saved).toEqual([]);
+    });
+
+    it('wraps a saved cursor past the end of a shrunken table back to the start', async () => {
+        rows.push(suspended());
+        const ctx = testJobContext({ afterId: 999 });
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([null]);
+    });
+
+    it('writes no cursor while there is nothing to page through', async () => {
+        const ctx = testJobContext();
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([]);
+    });
+
     it('leaves a suspension it did not cause alone', async () => {
         rows.push(
             durableRow({
@@ -283,7 +406,7 @@ describe('a suspension waiting on a balance', () => {
             }),
         );
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 });
@@ -298,9 +421,10 @@ describe('what a plan lets an account subscribe to', () => {
         await subscribe(appActor(`app-${seq}`));
 
         expect(created[0].limits).toEqual({
-            perUser: EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
-                DEFAULT_FREE_SUBSCRIPTION
-            ],
+            perUser:
+                EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
+                    DEFAULT_FREE_SUBSCRIPTION
+                ],
             perApp: EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP.bySubscription[
                 DEFAULT_FREE_SUBSCRIPTION
             ],