fix(homepage): serve the shell for /@<user>/<path>

Only /@<user> was routed, so a link to something inside a user's Public
folder fell through to the not-found redirect and landed on / without
opening anything. The GUI already opens the folder in Files, or the
file in its app, once the shell is served.

.github/workflows/notify-dev.yaml

@@ -0,0 +1,48 @@
+name: Notify HeyPuter (dev)
+
+on:
+  workflow_dispatch:
+    inputs:
+      puter_ref:
+        description: "puter branch, tag or SHA"
+        type: string
+        default: "main"
+
+permissions:
+  contents: read
+
+jobs:
+  authorize:
+    if: github.repository == 'HeyPuter/puter'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require maintain or admin
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ACTOR: ${{ github.triggering_actor }}
+        run: |
+          ROLE=$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$ACTOR/permission" --jq .role_name)
+          if [ "$ROLE" != "maintain" ] && [ "$ROLE" != "admin" ]; then
+            echo "::error::$ACTOR has role '$ROLE'; maintain or admin required"
+            exit 1
+          fi
+
+  notify:
+    needs: authorize
+    runs-on: ubuntu-latest
+    # Holds HEYPUTER_DISPATCH_TOKEN; limited to main so branch copies of this file can't read it.
+    environment: heyputer-dispatch
+    steps:
+      - name: Trigger heyputer dev deploy
+        env:
+          PUTER_REF: ${{ inputs.puter_ref }}
+          TOKEN: ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}
+        run: |
+          jq -nc --arg ref "$PUTER_REF" \
+            '{event_type: "puter-dev-requested", client_payload: {puter_ref: $ref}}' |
+          curl -fsS -X POST \
+            -H "Authorization: token $TOKEN" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/HeyPuter/heyputer/dispatches \
+            -d @-
+          echo "Requested \`$PUTER_REF\`" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/notify-prod.yaml

@@ -8,6 +8,7 @@ on:
 jobs:
   notify:
     runs-on: ubuntu-latest
+    environment: heyputer-dispatch
     steps:
       - name: Trigger heyputer build
         run: |
@@ -15,4 +16,4 @@ jobs:
             -H "Authorization: token ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}" \
             -H "Accept: application/vnd.github.v3+json" \
             https://api.github.com/repos/HeyPuter/heyputer/dispatches \
-            -d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'
+            -d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'

doc/pagination.md

@@ -63,6 +63,9 @@ store:
   sealed cursor: the last `id` it returned, and the last `(replaced_at, id)`
   among rows up to that id. Each is its own indexed read, merged into one
   page, so a row replaced after a client read it comes back from that cursor.
+  `from=end` with no cursor answers an empty page whose cursor holds the
+  slice's newest `id` and the slice's latest `(replaced_at, id)` as the
+  replacement mark.
 
 Cursors that carry a sort also pin it: a request that passes a cursor plus a
 conflicting sort is rejected with 400.

extensions/whoami.test.ts

@@ -257,8 +257,6 @@ describe('whoami extension — handleWhoami', () => {
         expect(body.created_ts).toBeGreaterThan(0);
         // `directories` is only sent to user actors — confirm it’s present.
         expect(body.directories).toBeDefined();
-        // taskbar_items is only sent to user actors.
-        expect(body).toHaveProperty('taskbar_items');
     });
 
     it('reports the SMS-to-card fallback only once a send has opened it', async () => {
@@ -322,7 +320,7 @@ describe('whoami extension — handleWhoami', () => {
         expect(flags.payment_bypass).toBeUndefined();
     });
 
-    it('strips desktop_bg_*, created_ts and human_readable_age fields for app actors', async () => {
+    it('strips created_ts and human_readable_age fields for app actors', async () => {
         const user = await seedUser();
         const { res, captured } = makeRes();
 
@@ -340,9 +338,6 @@ describe('whoami extension — handleWhoami', () => {
         expect(body.app_name).toBe('app-test-actor');
         // is_user_token is stripped for app actors.
         expect(body.is_user_token).toBeUndefined();
-        expect(body.desktop_bg_url).toBeUndefined();
-        expect(body.desktop_bg_color).toBeUndefined();
-        expect(body.desktop_bg_fit).toBeUndefined();
         expect(body.human_readable_age).toBeUndefined();
         // Account age, in either form, is not exposed to apps.
         expect(body.created_ts).toBeUndefined();
@@ -432,7 +427,7 @@ describe('whoami extension — handleWhoami', () => {
             });
         });
 
-        it('cannot add taskbar_items for an app actor, which core omits', async () => {
+        it('cannot add directories for an app actor, which core omits', async () => {
             const user = await seedUser();
             const body = await whoamiWithListener(
                 makeActor({
@@ -440,11 +435,11 @@ describe('whoami extension — handleWhoami', () => {
                     app: { uid: 'app-test-actor' },
                 }),
                 (_key, event) => {
-                    event.details.taskbar_items = [{ name: 'injected' }];
+                    event.details.directories = { '/injected': 'x' };
                 },
             );
 
-            expect(body).not.toHaveProperty('taskbar_items');
+            expect(body).not.toHaveProperty('directories');
         });
     });
 
@@ -525,7 +520,7 @@ describe('whoami extension — handleWhoami', () => {
             expect(body.otp_secret).toBeUndefined();
             expect(body.signup_ip).toBeUndefined();
             // Not just absent as a key — the values must not appear anywhere
-            // in the payload (nested under metadata, taskbar items, …).
+            // in the payload (nested under metadata, …).
             const serialized = JSON.stringify(body);
             expect(serialized).not.toContain('5551234567');
             expect(serialized).not.toContain('fp_ABC123');

extensions/whoami.ts

@@ -1,6 +1,5 @@
 import { Context } from '@heyputer/backend/src/core';
 import { extension } from '@heyputer/backend/src/extensions';
-import { APP_ICON_SIZES } from '@heyputer/backend/src/util/appIcon.js';
 import {
     buildUserDetails,
     scrubSensitive,
@@ -26,7 +25,7 @@ const APP_VISIBLE_LISTENER_KEYS: ReadonlySet<string> = new Set([
 ]);
 
 export const handleWhoami = async (
-    req: Request,
+    _req: Request,
     res: Response,
 ): Promise<void> => {
     const actor = Context.get('actor');
@@ -54,19 +53,10 @@ export const handleWhoami = async (
         return;
     }
 
-    const rawIconSize =
-        typeof req.query?.icon_size === 'string'
-            ? Number(req.query.icon_size)
-            : undefined;
-    const iconSize =
-        rawIconSize !== undefined && APP_ICON_SIZES.includes(rawIconSize)
-            ? rawIconSize
-            : undefined;
-
     const details = await buildUserDetails(
         user,
         { config: extension.config, clients, stores, services },
-        { isUser, iconSize, noIcons: !iconSize },
+        { isUser },
     );
     details.human_readable_age = user.timestamp
         ? timeago.format(new Date(user.timestamp as string))
@@ -81,9 +71,6 @@ export const handleWhoami = async (
             delete details.email;
             delete details.unconfirmed_email;
         }
-        delete details.desktop_bg_url;
-        delete details.desktop_bg_color;
-        delete details.desktop_bg_fit;
         delete details.human_readable_age;
         delete details.created_ts;
         delete details.is_user_token;
@@ -121,8 +108,7 @@ export const handleWhoami = async (
     }
 
     const subscription = details.subscription as
-        | { offering?: Record<string, unknown> }
-        | undefined;
+        { offering?: Record<string, unknown> } | undefined;
     if (subscription?.offering) {
         delete subscription.offering.group;
         delete subscription.offering.benefits;

src/backend/clients/database/DatabaseClient.test.ts

@@ -285,6 +285,18 @@ describe('AbstractDatabaseClient — SQL generation', () => {
         );
     });
 
+    it('reads the database clock as epoch milliseconds in each dialect', () => {
+        expect(sqlite().nowMsExpr()).toBe(
+            "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+        );
+        expect(mysql().nowMsExpr()).toBe(
+            "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        );
+        expect(postgres().nowMsExpr()).toBe(
+            'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+        );
+    });
+
     it('describes a stored timestamp the way each driver reads it back', () => {
         const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
         expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');

src/backend/clients/database/DatabaseClient.ts

@@ -401,6 +401,21 @@ export class AbstractDatabaseClient extends PuterClient {
         });
     }
 
+    /**
+     * The database's own clock as epoch milliseconds, so nodes whose clocks
+     * disagree still agree on it. The mysql form ignores the session time zone,
+     * so a DST change can't shift it.
+     */
+    nowMsExpr(): string {
+        return this.case({
+            sqlite: "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+            postgres:
+                'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+            otherwise:
+                "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        });
+    }
+
     /**
      * What a read of a timestamp column holding `seconds` returns: sqlite gives
      * back the UTC text it stored, mysql2 and pg a `Date` read in node's time

src/backend/clients/database/SqliteDatabaseClient.test.ts

@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
 import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
 
 /** Highest schema version the migration table can reach. */
-const CURRENT_SCHEMA_VERSION = 95;
+const CURRENT_SCHEMA_VERSION = 98;
 
 /**
  * These suites migrate real files on disk. Idle they finish in well under a
@@ -724,6 +724,85 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
         ]);
     });
 
+    it('keeps notification.send_key unique', async () => {
+        const rows = await client.read('PRAGMA index_list(`notification`)');
+        expect(
+            rows.find((r) => r.name === 'idx_notification_send_key'),
+        ).toMatchObject({ unique: 1 });
+        const columns = await client.read(
+            'PRAGMA index_info(`idx_notification_send_key`)',
+        );
+        expect(columns.map((c) => String(c.name))).toEqual(['send_key']);
+    });
+
+    it('creates event_user_handlers with its unique and holder indexes', async () => {
+        const columns = await client.read(
+            'PRAGMA table_info(`event_user_handlers`)',
+        );
+        expect(columns.map((c) => String(c.name))).toEqual([
+            'id',
+            'app_uid',
+            'holder_user_id',
+            'name',
+            'source',
+            'source_hash',
+            'created_at',
+            'updated_at',
+            'scope_since',
+        ]);
+        const indexes = await client.read(
+            'PRAGMA index_list(`event_user_handlers`)',
+        );
+        expect(
+            indexes.find(
+                (i) => i.name === 'idx_event_user_handlers_app_holder_name',
+            ),
+        ).toMatchObject({ unique: 1 });
+        expect(
+            indexes.find((i) => i.name === 'idx_event_user_handlers_holder_app'),
+        ).toMatchObject({ unique: 0 });
+        const columnsOf = async (index: string) =>
+            (await client.read(`PRAGMA index_info(\`${index}\`)`)).map((c) =>
+                String(c.name),
+            );
+        expect(await columnsOf('idx_event_user_handlers_app_holder_name')).toEqual(
+            ['app_uid', 'holder_user_id', 'name'],
+        );
+        expect(await columnsOf('idx_event_user_handlers_holder_app')).toEqual([
+            'holder_user_id',
+            'app_uid',
+        ]);
+    });
+
+    it('leaves event_handlers as it was', async () => {
+        const columns = await client.read('PRAGMA table_info(`event_handlers`)');
+        expect(columns.map((c) => String(c.name))).toEqual([
+            'id',
+            'app_uid',
+            'name',
+            'source',
+            'source_hash',
+            'created_at',
+            'updated_at',
+        ]);
+    });
+
+    it('adds event_subscriptions.handler_scope, app scope for existing rows', async () => {
+        const columns = await client.read(
+            'PRAGMA table_info(`event_subscriptions`)',
+        );
+        expect(columns.find((c) => c.name === 'handler_scope')).toMatchObject({
+            notnull: 1,
+            dflt_value: '0',
+        });
+        const indexes = await client.read(
+            'PRAGMA index_list(`event_subscriptions`)',
+        );
+        expect(indexes.map((i) => String(i.name))).toContain(
+            'idx_event_subscriptions_holder',
+        );
+    });
+
     it('leaves an already-migrated database untouched on a second boot', async () => {
         const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-'));
         const path = join(dir, 'nested', 'puter.sqlite');

src/backend/clients/database/SqliteDatabaseClient.ts

@@ -134,6 +134,9 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
     [92, ['0097_user-to-user-holder-permission-index.sql']],
     [93, ['0098_notification-groups.sql']],
     [94, ['0099_kv-share-handle-notify.sql']],
+    [95, ['0100_notification-send-key.sql']],
+    [96, ['0101_user-scoped-handlers.sql']],
+    [97, ['0102_notification-scope-id-index.sql']],
 ];
 
 export class SqliteDatabaseClient extends AbstractDatabaseClient {

src/backend/clients/database/migrations/mysql/mysql_mig_54.sql

@@ -0,0 +1,43 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`. See sqlite/0100_notification-send-key.sql for the
+-- rationale. The key is a uuid, so ascii and compared byte for byte. No
+-- per-file applied-state tracking, so every step is guarded and a replay does
+-- nothing.
+
+CALL _puter_add_col('notification', 'send_key', '`send_key` char(36) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
+DELIMITER //
+CREATE PROCEDURE _puter_add_notification_send_key_index()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_send_key'
+  ) THEN
+    ALTER TABLE `notification` ADD UNIQUE KEY `idx_notification_send_key`
+      (`send_key`), ALGORITHM=INPLACE, LOCK=NONE;
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_notification_send_key_index();
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;

src/backend/clients/database/migrations/mysql/mysql_mig_55.sql

@@ -0,0 +1,66 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Handlers an app's user published into their own scope, kept apart from
+-- `event_handlers` so code that predates them never reads them. See
+-- sqlite/0101_user-scoped-handlers.sql for the column rationale.
+--
+-- Idempotent: `CREATE TABLE IF NOT EXISTS` with the indexes declared inline,
+-- and a guarded column add. There is no per-file applied-state tracking, so a
+-- replay has to be a no-op.
+
+CREATE TABLE IF NOT EXISTS `event_user_handlers` (
+    `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+    -- Matches `apps`.`uid` exactly, charset included, so an equality against
+    -- one never falls back to a conversion.
+    `app_uid` char(40) CHARACTER SET latin1 COLLATE latin1_swedish_ci NOT NULL,
+    `holder_user_id` int unsigned NOT NULL,
+    `name` varchar(128) NOT NULL,
+    `source` mediumtext NOT NULL,
+    `source_hash` char(64) NOT NULL,
+    `created_at` bigint NOT NULL,
+    `updated_at` bigint NOT NULL,
+    -- When the scope first held a handler; kept across handler churn, so the
+    -- rent sweep sees the scope's age, not its oldest remaining handler's.
+    `scope_since` bigint NOT NULL,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `idx_event_user_handlers_app_holder_name` (`app_uid`, `holder_user_id`, `name`),
+    KEY `idx_event_user_handlers_holder_app` (`holder_user_id`, `app_uid`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- Guarded here rather than through `_puter_add_col`, so the file also applies
+-- on its own to a database that never ran the earlier migrations.
+DROP PROCEDURE IF EXISTS _puter_add_event_subscriptions_handler_scope;
+DELIMITER //
+CREATE PROCEDURE _puter_add_event_subscriptions_handler_scope()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'event_subscriptions'
+      AND COLUMN_NAME = 'handler_scope'
+  ) THEN
+    ALTER TABLE `event_subscriptions`
+      ADD COLUMN `handler_scope` tinyint unsigned NOT NULL DEFAULT 0,
+      ALGORITHM=INSTANT;
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_event_subscriptions_handler_scope();
+
+DROP PROCEDURE IF EXISTS _puter_add_event_subscriptions_handler_scope;

src/backend/clients/database/migrations/mysql/mysql_mig_56.sql

@@ -0,0 +1,41 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification` (`user_id`, `audience`, `app_uid`, `id`). See
+-- sqlite/0102_notification-scope-id-index.sql for the rationale. No per-file
+-- applied-state tracking, so the index is added only when missing and a
+-- replay does nothing.
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_scope_id_index;
+DELIMITER //
+CREATE PROCEDURE _puter_add_notification_scope_id_index()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_scope_id'
+  ) THEN
+    ALTER TABLE `notification` ADD INDEX `idx_notification_scope_id`
+      (`user_id`, `audience`, `app_uid`, `id`), ALGORITHM=INPLACE, LOCK=NONE;
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_notification_scope_id_index();
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_scope_id_index;

src/backend/clients/database/migrations/postgres/postgres_mig_43.sql

@@ -0,0 +1,24 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- notification.send_key. See sqlite/0100_notification-send-key.sql for the
+-- rationale. Idempotent via IF NOT EXISTS.
+
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS send_key char(36) DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_send_key
+    ON notification (send_key);

src/backend/clients/database/migrations/postgres/postgres_mig_44.sql

@@ -0,0 +1,39 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Handlers an app's user published into their own scope. See
+-- sqlite/0101_user-scoped-handlers.sql for the column rationale.
+-- Idempotent via IF NOT EXISTS.
+
+CREATE TABLE IF NOT EXISTS event_user_handlers (
+    id BIGSERIAL PRIMARY KEY,
+    app_uid VARCHAR(40) NOT NULL,
+    holder_user_id INTEGER NOT NULL,
+    name VARCHAR(128) NOT NULL,
+    source TEXT NOT NULL,
+    source_hash VARCHAR(64) NOT NULL,
+    created_at BIGINT NOT NULL,
+    updated_at BIGINT NOT NULL,
+    scope_since BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_event_user_handlers_app_holder_name
+    ON event_user_handlers (app_uid, holder_user_id, name);
+CREATE INDEX IF NOT EXISTS idx_event_user_handlers_holder_app
+    ON event_user_handlers (holder_user_id, app_uid);
+
+ALTER TABLE event_subscriptions ADD COLUMN IF NOT EXISTS handler_scope smallint NOT NULL DEFAULT 0;

src/backend/clients/database/migrations/postgres/postgres_mig_45.sql

@@ -0,0 +1,23 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- notification (user_id, audience, app_uid, id). See
+-- sqlite/0102_notification-scope-id-index.sql for the rationale. Idempotent via
+-- IF NOT EXISTS.
+
+CREATE INDEX IF NOT EXISTS idx_notification_scope_id
+    ON notification (user_id, audience, app_uid, id);

src/backend/clients/database/migrations/sqlite/0100_notification-send-key.sql

@@ -0,0 +1,27 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`: the key of the delayed send (a `deferred`
+-- schedule) that wrote this row or last replaced it, derived from the
+-- schedule and its slot. Unique, so one slot can never write two rows, and a
+-- send that crashed after writing finds its row again by it. Null on every row
+-- a delayed send never touched.
+
+ALTER TABLE `notification` ADD COLUMN `send_key` TEXT DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS `idx_notification_send_key`
+    ON `notification` (`send_key`);

src/backend/clients/database/migrations/sqlite/0101_user-scoped-handlers.sql

@@ -0,0 +1,48 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Handlers an app's user published into their own scope, in a table of their
+-- own so `event_handlers` stays the app owner's set. Same columns as
+-- `event_handlers` (see 0076_event-handlers.sql) plus the holder.
+--
+--   - `holder_user_id` : the user whose scope this is. A name is unique per
+--                        (app, holder), so the same name can live in the app's
+--                        set and in each holder's.
+--   - `(holder_user_id, app_uid)` : a holder's scopes, for listing and for the
+--                        rent count.
+--
+-- `event_subscriptions.handler_scope` says which table a row's `handler_name`
+-- refers to: 0 the app's set, 1 the row holder's scope.
+
+CREATE TABLE IF NOT EXISTS `event_user_handlers` (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "app_uid" TEXT NOT NULL,
+    "holder_user_id" INTEGER NOT NULL,
+    "name" TEXT NOT NULL,
+    "source" TEXT NOT NULL,
+    "source_hash" TEXT NOT NULL,
+    "created_at" INTEGER NOT NULL,    -- unix seconds
+    "updated_at" INTEGER NOT NULL,
+    "scope_since" INTEGER NOT NULL    -- unix seconds; the scope's first handler, kept across churn
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS `idx_event_user_handlers_app_holder_name`
+    ON `event_user_handlers` (`app_uid`, `holder_user_id`, `name`);
+CREATE INDEX IF NOT EXISTS `idx_event_user_handlers_holder_app`
+    ON `event_user_handlers` (`holder_user_id`, `app_uid`);
+
+ALTER TABLE `event_subscriptions` ADD COLUMN `handler_scope` INTEGER NOT NULL DEFAULT 0;

src/backend/clients/database/migrations/sqlite/0102_notification-scope-id-index.sql

@@ -0,0 +1,22 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Index a mailbox slice by row id, so a fetch finds the newest row of a slice,
+-- and pages forward through one, with a seek rather than a scan of the slice.
+
+CREATE INDEX IF NOT EXISTS `idx_notification_scope_id`
+    ON `notification` (`user_id`, `audience`, `app_uid`, `id`);

src/backend/clients/event/types.ts

@@ -581,6 +581,8 @@ export type EventMap = {
         updated?: true;
         /** Names this one send, for a receipt asked about it. */
         sendId: string;
+        /** The same `sendId` may be emitted again, as a delayed send is. */
+        replayable?: true;
     };
     /**
      * One notification, addressed at whatever sockets each region holds for the
@@ -664,9 +666,18 @@ export type EventMap = {
     // and going away (1→0, whether by removing the last handler or by the
     // destroy route). `actor.user` is the app's owner, not the caller — a
     // developer session publishing for an app it owns is the common case, but
-    // billing follows ownership.
-    'events.worker.create': { actor: Actor; appUid: string };
-    'events.worker.destroy': { actor: Actor; appUid: string };
+    // billing follows ownership. For `scope: 'user'` the unit is one user's
+    // handlers in one app instead, and `actor.user` is that user.
+    'events.worker.create': {
+        actor: Actor;
+        appUid: string;
+        scope: 'app' | 'user';
+    };
+    'events.worker.destroy': {
+        actor: Actor;
+        appUid: string;
+        scope: 'app' | 'user';
+    };
 
     // ---- Outer / GUI broadcast ----
     'outer.cacheUpdate': {
@@ -1030,11 +1041,10 @@ export type EventKey = keyof EventMap & string;
 // Generates a wildcard for every non-final dot-separated prefix of K.
 export type WildcardPrefixes<K extends string> =
     K extends `${infer Head}.${infer Tail}`
-        ?
-              | `${Head}.*`
-              | (Tail extends `${string}.${string}`
-                    ? `${Head}.${WildcardPrefixes<Tail>}`
-                    : never)
+        ? | `${Head}.*`
+          | (Tail extends `${string}.${string}`
+                ? `${Head}.${WildcardPrefixes<Tail>}`
+                : never)
         : never;
 
 export type ListenKey = EventKey | WildcardPrefixes<EventKey>;
@@ -1045,10 +1055,7 @@ export type MatchingEvents<P extends ListenKey> = P extends `${infer Prefix}.*`
 
 /** Acked, not acked, no socket anywhere, or sockets only in a peer region. */
 export type NotifDeliveryOutcome =
-    | 'received'
-    | 'unreceived'
-    | 'no-socket'
-    | 'remote-only';
+    'received' | 'unreceived' | 'no-socket' | 'remote-only';
 
 /** One socket's receipt for a live notification. */
 export interface NotifDeliveryResponder {

src/backend/clients/events/EventsWorkerInvokerClient.test.ts

@@ -23,18 +23,25 @@
  */
 
 import type { fetch as undiciFetch } from 'undici';
-import { describe, expect, it, vi } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
 import type { IConfig } from '../../types.js';
 import {
     DispatcherInvokeTransport,
     EVENTS_DEPLOYED_HEADER,
+    EVENTS_DEADLINE_MARGIN_MS,
+    EVENTS_DISPATCH_BATCH_PATH,
     EVENTS_DISPATCH_ERROR_HEADER,
     EVENTS_DISPATCH_PATH,
     EVENTS_ERROR_HEADER,
     EVENTS_HANDLED_HEADER,
+    EVENTS_HOLDER_HEADER,
+    EVENTS_INVOKE_TIMEOUT_MS,
+    EVENTS_WORKER_LEASE_FACTOR,
     EventsWorkerInvokerClient,
     INVOKE_CONNECTIONS,
+    SCHEDULE_INVOKE_CONNECTIONS,
     outcomeForStatus,
+    parseAnswerLine,
     type WorkerInvokeRequest,
 } from './EventsWorkerInvokerClient.js';
 
@@ -73,10 +80,7 @@ const refusedFetch: FetchImpl = (async () => {
 }) as unknown as FetchImpl;
 
 /** A stub fetch that records the URL it was called with. */
-const capturingFetch = (
-    calls: string[],
-    status = 200,
-): FetchImpl =>
+const capturingFetch = (calls: string[], status = 200): FetchImpl =>
     (async (url: string) => {
         calls.push(url);
         return new Response(null, { status });
@@ -84,9 +88,13 @@ const capturingFetch = (
 
 describe('DispatcherInvokeTransport', () => {
     it('reports a handled 400 with no error — the handler`s own refusal', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(400, { [EVENTS_HANDLED_HEADER]: '1' }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(400, { [EVENTS_HANDLED_HEADER]: '1' }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 400,
             handled: true,
@@ -94,9 +102,13 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('reports an unmarked 404 with no dispatch-error header as unhandled', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(404),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(404),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 404,
             handled: false,
@@ -104,9 +116,15 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('turns a dispatch-error header into a null status with the reason', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(502, { [EVENTS_DISPATCH_ERROR_HEADER]: 'deploy-failed' }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(502, {
+                    [EVENTS_DISPATCH_ERROR_HEADER]: 'deploy-failed',
+                }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: null,
             error: 'dispatcher: deploy-failed (502)',
@@ -120,9 +138,13 @@ describe('DispatcherInvokeTransport', () => {
             headers.push(init?.headers as Record<string, string>);
             return new Response(null, { status: 200 });
         }) as unknown as FetchImpl;
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl,
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl,
+            },
+        );
 
         await transport.send(CALL);
         await transport.send({ ...CALL, deployed: true });
@@ -132,9 +154,13 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('reports a handled 200 as settled with no error', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(200, { [EVENTS_HANDLED_HEADER]: '1' }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(200, { [EVENTS_HANDLED_HEADER]: '1' }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 200,
             handled: true,
@@ -142,9 +168,13 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('passes a 429 through as-is', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(429),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(429),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 429,
             handled: false,
@@ -152,18 +182,26 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('answers null and says so when the request times out', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: hangingFetch,
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: hangingFetch,
+            },
+        );
         const result = await transport.send({ ...CALL, timeoutMs: 10 });
         expect(result.status).toBeNull();
         expect(result.timedOut).toBe(true);
     });
 
     it('answers null without a timeout when the request fails outright', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: refusedFetch,
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: refusedFetch,
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: null,
             error: 'fetch failed',
@@ -187,12 +225,16 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('surfaces the runtime`s own error header alongside its status', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(500, {
-                [EVENTS_HANDLED_HEADER]: '1',
-                [EVENTS_ERROR_HEADER]: 'handler-threw',
-            }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(500, {
+                    [EVENTS_HANDLED_HEADER]: '1',
+                    [EVENTS_ERROR_HEADER]: 'handler-threw',
+                }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 500,
             handled: true,
@@ -208,7 +250,9 @@ describe('DispatcherInvokeTransport', () => {
             { fetchImpl: capturingFetch(calls) },
         );
         await transport.send(CALL);
-        expect(calls).toEqual([`http://dispatcher/prefix${EVENTS_DISPATCH_PATH}`]);
+        expect(calls).toEqual([
+            `http://dispatcher/prefix${EVENTS_DISPATCH_PATH}`,
+        ]);
     });
 });
 
@@ -237,10 +281,12 @@ describe('DispatcherInvokeTransport with every connection busy', () => {
     /** Every connection taken by a call nobody has answered yet. */
     const saturated = async () => {
         const { fetchImpl, held } = heldFetch();
+        // Slots are per call, batched or not; one request per call keeps
+        // the count readable.
         const transport = new DispatcherInvokeTransport(
             'http://dispatcher',
             's',
-            { fetchImpl },
+            { fetchImpl, batchSize: 1 },
         );
         const busy = Array.from({ length: INVOKE_CONNECTIONS }, () =>
             transport.send(CALL),
@@ -303,6 +349,534 @@ describe('DispatcherInvokeTransport with every connection busy', () => {
     });
 });
 
+describe('DispatcherInvokeTransport schedule lane', () => {
+    const heldFetch = () => {
+        const held: Array<{ answer: () => void; body: string }> = [];
+        const fetchImpl = ((_url: string, init: RequestInit) =>
+            new Promise((resolve) => {
+                held.push({
+                    body: String(init.body),
+                    answer: () =>
+                        resolve(
+                            new Response(null, {
+                                status: 200,
+                                headers: { [EVENTS_HANDLED_HEADER]: '1' },
+                            }),
+                        ),
+                });
+            })) as unknown as FetchImpl;
+        return { fetchImpl, held };
+    };
+
+    it('holds schedule runs to their share and keeps serving deliveries', async () => {
+        const { fetchImpl, held } = heldFetch();
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            { fetchImpl, batchSize: 1 },
+        );
+        const scheduled = Array.from(
+            { length: SCHEDULE_INVOKE_CONNECTIONS + 5 },
+            () => transport.send({ ...CALL, lane: 'schedule' as const }),
+        );
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS),
+        );
+        // A delivery waiting behind the schedule queue still gets a connection.
+        const delivery = transport.send(CALL);
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 1),
+        );
+        held[SCHEDULE_INVOKE_CONNECTIONS].answer();
+        await expect(delivery).resolves.toMatchObject({ status: 200 });
+
+        // A schedule run's slot goes to the next schedule run waiting.
+        held[0].answer();
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 2),
+        );
+        for (let i = 0; i < 4; i++) held[i + 1].answer();
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 6),
+        );
+        for (const call of held) call.answer();
+        await Promise.all(scheduled);
+    });
+
+    it('builds the body after the connection wait, so a stamp is the send time', async () => {
+        const { fetchImpl, held } = heldFetch();
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            { fetchImpl, batchSize: 1 },
+        );
+        const busy = Array.from({ length: SCHEDULE_INVOKE_CONNECTIONS }, () =>
+            transport.send({ ...CALL, lane: 'schedule' as const }),
+        );
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS),
+        );
+        let renders = 0;
+        const stamped = transport.send({
+            ...CALL,
+            lane: 'schedule',
+            render: () => `{"n":${++renders}}`,
+        });
+        held[0].answer();
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 1),
+        );
+        expect(held[SCHEDULE_INVOKE_CONNECTIONS].body).toBe('{"n":1}');
+        for (const call of held) call.answer();
+        await Promise.all([...busy, stamped]);
+    });
+});
+
+describe('DispatcherInvokeTransport batches', () => {
+    interface BatchRequest {
+        url: string;
+        body: { calls: Array<Record<string, unknown>> };
+        signal: AbortSignal;
+        /** Streams one answer line back. */
+        answer: (line: Record<string, unknown>) => void;
+        end: () => void;
+    }
+
+    /**
+     * A fetch whose batch answers stream only when told to; single calls get a
+     * handled 200 straight away.
+     */
+    const streamingFetch = () => {
+        const batches: BatchRequest[] = [];
+        const singles: Array<{ url: string; headers: Record<string, string> }> =
+            [];
+        const fetchImpl = (async (url: string, init: RequestInit) => {
+            if (!url.endsWith(EVENTS_DISPATCH_BATCH_PATH)) {
+                singles.push({
+                    url,
+                    headers: init.headers as Record<string, string>,
+                });
+                return new Response(null, {
+                    status: 200,
+                    headers: { [EVENTS_HANDLED_HEADER]: '1' },
+                });
+            }
+            let controller!: ReadableStreamDefaultController<Uint8Array>;
+            const stream = new ReadableStream<Uint8Array>({
+                start: (c) => {
+                    controller = c;
+                },
+            });
+            const encoder = new TextEncoder();
+            init.signal?.addEventListener('abort', () => {
+                try {
+                    controller.error(init.signal!.reason);
+                } catch {
+                    // already closed
+                }
+            });
+            batches.push({
+                url,
+                body: JSON.parse(String(init.body)),
+                signal: init.signal!,
+                answer: (line) =>
+                    controller.enqueue(
+                        encoder.encode(`${JSON.stringify(line)}\n`),
+                    ),
+                end: () => controller.close(),
+            });
+            return new Response(stream, {
+                status: 200,
+                headers: { 'content-type': 'application/x-ndjson' },
+            });
+        }) as unknown as FetchImpl;
+        return { fetchImpl, batches, singles };
+    };
+
+    const transportFor = (
+        fetchImpl: FetchImpl,
+        options: { batchSize?: number; batchWindowMs?: number } = {},
+    ) =>
+        new DispatcherInvokeTransport('http://dispatcher', 's', {
+            fetchImpl,
+            ...options,
+        });
+
+    it('sends calls made together as one request, and answers each from its own line', async () => {
+        const { fetchImpl, batches, singles } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const first = transport.send({ ...CALL, script: 'evw-a', key: 'k1:a' });
+        const second = transport.send({
+            ...CALL,
+            script: 'evw-b',
+            deployed: true,
+            body: '{"b":1}',
+        });
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+
+        expect(batches[0].url).toBe(
+            `http://dispatcher${EVENTS_DISPATCH_BATCH_PATH}`,
+        );
+        expect(batches[0].body.calls).toEqual([
+            {
+                script: 'evw-a',
+                appUid: 'app-test',
+                key: 'k1:a',
+                body: '{}',
+                waitMs: expect.any(Number),
+            },
+            {
+                script: 'evw-b',
+                appUid: 'app-test',
+                key: 'k1:test',
+                deployed: true,
+                body: '{"b":1}',
+                waitMs: expect.any(Number),
+            },
+        ]);
+        // What is left of half the timeout, the share a call may wait.
+        expect(batches[0].body.calls[0].waitMs).toBeLessThanOrEqual(2_500);
+        expect(batches[0].body.calls[0].waitMs).toBeGreaterThan(2_400);
+
+        // Out of order: each line names the call it answers.
+        batches[0].answer({ i: 1, status: 400, handled: true });
+        await expect(second).resolves.toEqual({ status: 400, handled: true });
+        batches[0].answer({
+            i: 0,
+            status: 503,
+            handled: true,
+            error: 'platform-error',
+            retryAfter: '3',
+        });
+        await expect(first).resolves.toEqual({
+            status: 503,
+            handled: true,
+            error: 'platform-error',
+            retryAfterMs: 3000,
+        });
+        // Every call answered, so the request is let go.
+        expect(batches[0].signal.aborted).toBe(true);
+        expect(singles).toEqual([]);
+    });
+
+    it('sends a call with nobody to batch with on the single route', async () => {
+        const { fetchImpl, batches, singles } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        await expect(transport.send(CALL)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(batches).toEqual([]);
+        expect(singles[0].url).toBe(`http://dispatcher${EVENTS_DISPATCH_PATH}`);
+    });
+
+    it('sends a full batch at once, without waiting out the window', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl, {
+            batchSize: 2,
+            batchWindowMs: 60_000,
+        });
+        const sent = [transport.send(CALL), transport.send(CALL)];
+        const third = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        expect(batches[0].body.calls).toHaveLength(2);
+        batches[0].answer({ i: 0, status: 200, handled: true });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await Promise.all(sent);
+        transport.close();
+        await expect(third).resolves.toMatchObject({ status: 200 });
+    });
+
+    it('gives each call its own timeout, without failing the rest of the batch', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const quick = transport.send({ ...CALL, timeoutMs: 50 });
+        const slow = transport.send({ ...CALL, timeoutMs: 5_000 });
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, started: true });
+        batches[0].answer({ i: 1, started: true });
+
+        await expect(quick).resolves.toMatchObject({
+            status: null,
+            timedOut: true,
+        });
+        expect(batches[0].signal.aborted).toBe(false);
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await expect(slow).resolves.toEqual({ status: 200, handled: true });
+    });
+
+    it('runs a call`s timeout from its started line, not from the send', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        let settled = false;
+        const queued = transport
+            .send({ ...CALL, timeoutMs: 200 })
+            .then((answer) => {
+                settled = true;
+                return answer;
+            });
+        const other = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+
+        // Queued behind other calls for longer than its whole timeout.
+        await new Promise((resolve) => setTimeout(resolve, 250));
+        expect(settled).toBe(false);
+        batches[0].answer({ i: 0, started: true });
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        expect(settled).toBe(false);
+        await expect(queued).resolves.toMatchObject({
+            status: null,
+            timedOut: true,
+        });
+
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await other;
+    });
+
+    it('puts a call that never started on Puter, not on its handler', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const never = transport.send({ ...CALL, timeoutMs: 20 });
+        const queueTimedOut = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 1, status: 503, dispatch: 'queue-timeout' });
+
+        await expect(queueTimedOut).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: queue-timeout (503)',
+            dispatchReason: 'queue-timeout',
+        });
+        // Its wait plus the grace for the started line to arrive.
+        await expect(never).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: call never started',
+        });
+        const client = new EventsWorkerInvokerClient({} as IConfig);
+        client.setTransport({ send: async () => queueTimedOut });
+        await expect(client.invoke(REQUEST)).resolves.toMatchObject({
+            outcome: 'unavailable',
+        });
+    });
+
+    it('puts a call the stream ended without answering on Puter', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const answered = transport.send(CALL);
+        const dropped = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, status: 200, handled: true });
+        batches[0].end();
+
+        await expect(answered).resolves.toEqual({ status: 200, handled: true });
+        await expect(dropped).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: batch ended without an answer',
+        });
+    });
+
+    it('carries a dispatch failure in a line as that call`s reason', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const missing = transport.send(CALL);
+        const other = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, status: 404, dispatch: 'missing' });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+
+        await expect(missing).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: missing (404)',
+            dispatchReason: 'missing',
+        });
+        await expect(other).resolves.toMatchObject({ status: 200 });
+    });
+
+    it('fails every call of a batch the dispatcher refused, or something else answered', async () => {
+        for (const [response, expected] of [
+            [
+                new Response('forbidden', {
+                    status: 403,
+                    headers: { [EVENTS_DISPATCH_ERROR_HEADER]: 'forbidden' },
+                }),
+                {
+                    status: null,
+                    error: 'dispatcher: forbidden (403)',
+                    dispatchReason: 'forbidden',
+                },
+            ],
+            [
+                new Response('not found', { status: 404 }),
+                {
+                    status: null,
+                    error: 'dispatcher: unmarked 404 for a batch',
+                },
+            ],
+        ] as const) {
+            const transport = transportFor(
+                (async () => response) as unknown as FetchImpl,
+            );
+            const answers = await Promise.all([
+                transport.send(CALL),
+                transport.send(CALL),
+            ]);
+            expect(answers).toEqual([expected, expected]);
+        }
+    });
+
+    it('fails every call when the batch request itself fails', async () => {
+        const transport = transportFor(refusedFetch);
+        const answers = await Promise.all([
+            transport.send(CALL),
+            transport.send(CALL),
+        ]);
+        expect(answers).toEqual([
+            { status: null, error: 'fetch failed' },
+            { status: null, error: 'fetch failed' },
+        ]);
+    });
+
+    it('falls back to single calls on a dispatcher without the batch route', async () => {
+        const urls: string[] = [];
+        const fetchImpl = (async (url: string) => {
+            urls.push(url);
+            if (url.endsWith(EVENTS_DISPATCH_BATCH_PATH))
+                return new Response('no-such-route', {
+                    status: 404,
+                    headers: {
+                        [EVENTS_DISPATCH_ERROR_HEADER]: 'no-such-route',
+                    },
+                });
+            return new Response(null, {
+                status: 200,
+                headers: { [EVENTS_HANDLED_HEADER]: '1' },
+            });
+        }) as unknown as FetchImpl;
+        const transport = transportFor(fetchImpl);
+
+        const first = await Promise.all([
+            transport.send(CALL),
+            transport.send(CALL),
+        ]);
+        expect(first).toEqual([
+            { status: 200, handled: true },
+            { status: 200, handled: true },
+        ]);
+        // The next pair goes straight to the single route.
+        await Promise.all([transport.send(CALL), transport.send(CALL)]);
+        expect(urls.map((url) => new URL(url).pathname)).toEqual([
+            EVENTS_DISPATCH_BATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+        ]);
+    });
+
+    it('builds each body when the batch goes out', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        let renders = 0;
+        const sent = [
+            transport.send({ ...CALL, render: () => `{"n":${++renders}}` }),
+            transport.send(CALL),
+        ];
+        expect(renders).toBe(0);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        expect(batches[0].body.calls[0].body).toBe('{"n":1}');
+        batches[0].answer({ i: 0, status: 200, handled: true });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await Promise.all(sent);
+    });
+
+    it('deploys a call`s missing script and retries it, batched with the rest', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const client = new EventsWorkerInvokerClient({
+            events: { invokeTimeoutMs: 5_000 },
+        } as unknown as IConfig);
+        client.setTransport(transportFor(fetchImpl));
+        const missHandler = vi.fn(async () => 'deployed' as const);
+        client.setMissHandler(missHandler);
+
+        const missing = client.invoke({ ...REQUEST, script: 'evw-gone' });
+        const resident = client.invoke(REQUEST);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, status: 404, dispatch: 'missing' });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await expect(resident).resolves.toMatchObject({ outcome: 'settled' });
+
+        // The retry is alone, so it goes on the single route.
+        await expect(missing).resolves.toMatchObject({ outcome: 'settled' });
+        expect(missHandler).toHaveBeenCalledWith('app-test', 'evw-gone', {
+            lane: undefined,
+        });
+    });
+
+    it('reads only well-formed answer lines', () => {
+        expect(parseAnswerLine('not json')).toBeNull();
+        expect(parseAnswerLine('{"status":200}')).toBeNull();
+        expect(parseAnswerLine('{"i":-1,"status":200}')).toBeNull();
+        expect(parseAnswerLine('{"i":0}')).toBeNull();
+        expect(parseAnswerLine('{"i":1,"started":true}')).toEqual({
+            i: 1,
+            started: true,
+        });
+        expect(parseAnswerLine('{"i":2,"status":200}')).toEqual({
+            i: 2,
+            answer: { status: 200, handled: false },
+        });
+    });
+});
+
+describe('EventsWorkerInvokerClient invoke options', () => {
+    it('stamps the event as each attempt goes out', async () => {
+        const bodies: string[] = [];
+        const client = makeClient();
+        client.setTransport({
+            send: async (call) => {
+                bodies.push(call.body);
+                return { status: 200, handled: true };
+            },
+        });
+        let n = 0;
+        await client.invoke(REQUEST, {
+            stampEvent: (event) => ({ ...(event as object), sentAt: ++n }),
+        });
+        expect(JSON.parse(bodies[0]).event).toEqual({ sentAt: 1 });
+    });
+
+    it('gives a call up as Puter`s when its deploy is put off for another', async () => {
+        const headersSeen: Array<Record<string, string>> = [];
+        const client = makeClient();
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl: missUntilDeployedFetch(headersSeen),
+            }),
+        );
+        client.setMissHandler(async () => 'busy');
+        await expect(client.invoke(REQUEST)).resolves.toMatchObject({
+            outcome: 'unavailable',
+            deployBusy: true,
+        });
+        expect(headersSeen).toHaveLength(1);
+    });
+
+    it('tells the miss handler whose call needs the deploy', async () => {
+        const lanes: unknown[] = [];
+        const client = makeClient();
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl: missUntilDeployedFetch([]),
+            }),
+        );
+        client.setMissHandler(async (_appUid, _script, options) => {
+            lanes.push(options.lane);
+            return 'deployed';
+        });
+        await client.invoke(REQUEST, { lane: 'schedule' });
+        await client.invoke(REQUEST);
+        expect(lanes).toEqual(['schedule', undefined]);
+    });
+});
+
 const REQUEST: WorkerInvokeRequest = {
     script: 'evw-test',
     appUid: 'app-test',
@@ -313,7 +887,8 @@ const REQUEST: WorkerInvokeRequest = {
     ctx: {},
 };
 
-const makeClient = () => new EventsWorkerInvokerClient({} as unknown as IConfig);
+const makeClient = () =>
+    new EventsWorkerInvokerClient({} as unknown as IConfig);
 
 /** Answers `missing` until it sees the deployed header, then settles. */
 const missUntilDeployedFetch = (
@@ -731,3 +1306,217 @@ describe('outcomeForStatus', () => {
         },
     );
 });
+
+describe('EventsWorkerInvokerClient overall deadline', () => {
+    afterEach(() => {
+        vi.useRealTimers();
+    });
+
+    const LEASE_MS = EVENTS_INVOKE_TIMEOUT_MS * EVENTS_WORKER_LEASE_FACTOR;
+
+    /**
+     * A transport on a fake clock: each send costs what `cost` says, and
+     * answers a miss until the deployed header, then a handled 200.
+     */
+    const timedTransport = (cost: (deployed: boolean) => number) => {
+        const sends: Array<{ deployed: boolean; timeoutMs: number }> = [];
+        const transport = {
+            send: async (call: {
+                deployed?: boolean;
+                timeoutMs: number;
+            }): Promise<unknown> => {
+                const deployed = call.deployed === true;
+                sends.push({ deployed, timeoutMs: call.timeoutMs });
+                // A send never outlasts its own timeout.
+                vi.advanceTimersByTime(
+                    Math.min(cost(deployed), call.timeoutMs * 1.5),
+                );
+                return deployed
+                    ? { status: 200, handled: true }
+                    : {
+                          status: null,
+                          dispatchReason: 'missing',
+                          error: 'dispatcher: missing (404)',
+                      };
+            },
+        };
+        return { transport, sends };
+    };
+
+    const run = async (
+        cost: (deployed: boolean) => number,
+        deployMs: number,
+    ) => {
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const { transport, sends } = timedTransport(cost);
+        const client = makeClient();
+        client.setTransport(transport as never);
+        client.setMissHandler(async () => {
+            vi.advanceTimersByTime(deployMs);
+            return 'deployed';
+        });
+        const started = Date.now();
+        const result = await client.invoke(REQUEST);
+        return { result, sends, elapsed: Date.now() - started };
+    };
+
+    it('leaves a fast miss as it was: a full-timeout resend', async () => {
+        const { result, sends } = await run(() => 100, 2_000);
+        expect(result).toEqual({ outcome: 'settled', status: 200 });
+        expect(sends).toEqual([
+            { deployed: false, timeoutMs: EVENTS_INVOKE_TIMEOUT_MS },
+            { deployed: true, timeoutMs: EVENTS_INVOKE_TIMEOUT_MS },
+        ]);
+    });
+
+    it('shortens the resend after a slow deploy, so the call ends before the lease', async () => {
+        // A miss that took 12 s, a deploy that took 30 s, and a resend that
+        // would use everything it is given.
+        const { result, sends, elapsed } = await run(
+            (deployed) => (deployed ? 1e9 : 12_000),
+            30_000,
+        );
+        expect(result.outcome).toBe('settled');
+        expect(sends[1].deployed).toBe(true);
+        expect(sends[1].timeoutMs).toBeLessThan(EVENTS_INVOKE_TIMEOUT_MS);
+        expect(elapsed).toBeLessThanOrEqual(
+            LEASE_MS - EVENTS_DEADLINE_MARGIN_MS,
+        );
+    });
+
+    it('gives up on a deploy that never settles before the lease, leaving it running', async () => {
+        vi.useFakeTimers();
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        const { transport, sends } = timedTransport(() => 0);
+        const client = makeClient();
+        client.setTransport(transport as never);
+        let rejectDeploy!: (err: Error) => void;
+        client.setMissHandler(
+            () =>
+                new Promise<string>((_resolve, reject) => {
+                    rejectDeploy = reject;
+                }),
+        );
+        let finished: unknown = null;
+        const pending = client.invoke(REQUEST).then((r) => (finished = r));
+
+        // Up to where the call must have ended, and no further.
+        await vi.advanceTimersByTimeAsync(LEASE_MS - EVENTS_DEADLINE_MARGIN_MS);
+        expect(finished).toMatchObject({
+            outcome: 'unavailable',
+            status: null,
+        });
+        await pending;
+        expect(sends).toHaveLength(1);
+        // The deploy was not cancelled; its late failure is logged, not thrown.
+        rejectDeploy(new Error('stalled'));
+        await vi.advanceTimersByTimeAsync(0);
+        expect(warn).toHaveBeenCalledWith(
+            '[events] deploy failed late',
+            expect.any(Error),
+        );
+        warn.mockRestore();
+    });
+
+    it('does not resend when the deploy leaves less than the margin', async () => {
+        const { result, sends } = await run(
+            () => 100,
+            LEASE_MS - EVENTS_DEADLINE_MARGIN_MS * 2 + 1_000,
+        );
+        expect(sends).toHaveLength(1);
+        expect(result).toMatchObject({ outcome: 'unavailable', status: null });
+    });
+});
+
+describe('the holder of a user scope on the wire', () => {
+    const HOLDER = 'user-uuid-1';
+
+    it('sends the holder header on a single invoke, and none for an app scope', async () => {
+        const seen: Array<Record<string, string>> = [];
+        const fetchImpl = (async (_url: string, init?: RequestInit) => {
+            seen.push((init?.headers ?? {}) as Record<string, string>);
+            return new Response(null, {
+                status: 200,
+                headers: { [EVENTS_HANDLED_HEADER]: '1' },
+            });
+        }) as unknown as FetchImpl;
+        const client = makeClient();
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl,
+                batchSize: 1,
+            }),
+        );
+
+        await client.invoke({ ...REQUEST, holder: HOLDER });
+        await client.invoke(REQUEST);
+
+        expect(seen[0][EVENTS_HOLDER_HEADER]).toBe(HOLDER);
+        expect(EVENTS_HOLDER_HEADER in seen[1]).toBe(false);
+    });
+
+    it('puts a holder field on a batch call of a user scope only', async () => {
+        const bodies: Array<{ calls: Array<Record<string, unknown>> }> = [];
+        const fetchImpl = (async (_url: string, init?: RequestInit) => {
+            bodies.push(JSON.parse(String(init?.body)));
+            return new Response(
+                '{"i":0,"status":200,"handled":true}\n{"i":1,"status":200,"handled":true}\n',
+                { status: 200 },
+            );
+        }) as unknown as FetchImpl;
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            { fetchImpl },
+        );
+
+        await Promise.all([
+            transport.send({ ...CALL, holder: HOLDER }),
+            transport.send(CALL),
+        ]);
+
+        expect(bodies).toHaveLength(1);
+        expect(bodies[0].calls[0].holder).toBe(HOLDER);
+        expect('holder' in bodies[0].calls[1]).toBe(false);
+    });
+
+    it('hands the holder to the miss handler, and nothing for an app scope', async () => {
+        const seen: Array<{ holder?: string }> = [];
+        const client = makeClient();
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl: missUntilDeployedFetch([]),
+                batchSize: 1,
+            }),
+        );
+        client.setMissHandler(async (_appUid, _script, options) => {
+            seen.push(options);
+            return 'deployed';
+        });
+
+        await client.invoke({ ...REQUEST, holder: HOLDER });
+        await client.invoke(REQUEST);
+
+        expect(seen[0].holder).toBe(HOLDER);
+        expect('holder' in seen[1]).toBe(false);
+    });
+
+    it('keeps the holder out of the body the script sees', async () => {
+        const bodies: string[] = [];
+        const client = makeClient();
+        client.setTransport({
+            send: async (call) => {
+                bodies.push(call.body);
+                expect(call.holder).toBe(HOLDER);
+                return { status: 200, handled: true };
+            },
+        });
+        await client.invoke({ ...REQUEST, holder: HOLDER });
+        expect(Object.keys(JSON.parse(bodies[0])).sort()).toEqual([
+            'ctx',
+            'event',
+            'handler',
+            'token',
+        ]);
+    });
+});

src/backend/clients/events/EventsWorkerInvokerClient.ts

@@ -35,6 +35,7 @@ import { PuterClient } from '../types.js';
  *     x-puter-internal-auth:  <internal secret>
  *     x-puter-events-script:  <script the handler set deploys as>
  *     x-puter-events-app:     <app uid, for the deploy-on-miss callback>
+ *     x-puter-events-holder:  <holder uuid of a user scope; absent for app scope>
  *     x-puter-events-key:     <key derived for that script>
  *     { handler, event, ctx, token }
  *
@@ -43,6 +44,20 @@ import { PuterClient } from '../types.js';
  *
  * On a miss, a handler set here may deploy the script itself and retry once
  * with `x-puter-events-deployed: 1`, skipping the dispatcher's own callback.
+ *
+ * Calls that come due within a few milliseconds of each other go out together:
+ *
+ *     POST <events dispatcher>/invoke-batch
+ *     x-puter-internal-auth:  <internal secret>
+ *     { calls: [{ script, appUid, holder?, key, deployed?, body, waitMs }] }
+ *
+ * The dispatcher runs a few at a time, in order, and streams NDJSON: `{ i,
+ * started: true }` when a call goes to its worker, then `{ i, status, handled?,
+ * error?, retryAfter?, dispatch? }` when it finishes — the same facts the
+ * single route carries in its status and headers. A call's timeout runs from
+ * its `started` line, so time spent queued behind other calls is never the
+ * handler's. A call still queued `waitMs` after the batch arrived is never
+ * started and answers `dispatch: "queue-timeout"`.
  */
 
 /** The one route an events worker answers. */
@@ -51,6 +66,27 @@ export const EVENTS_INVOKE_PATH = '/__events/invoke';
 /** The route the events dispatcher takes an invocation on. */
 export const EVENTS_DISPATCH_PATH = '/invoke';
 
+/** The route it takes several on, answering each as it finishes. */
+export const EVENTS_DISPATCH_BATCH_PATH = '/invoke-batch';
+
+/** Calls one batch carries at most. */
+export const INVOKE_BATCH_SIZE = 60;
+
+/** How long the first call of a batch waits for others to join it. */
+export const INVOKE_BATCH_WINDOW_MS = 5;
+
+/**
+ * How long a dispatcher without the batch route is sent single calls before the
+ * batch route is tried again.
+ */
+const BATCH_UNSUPPORTED_RETRY_MS = 5 * 60_000;
+
+/**
+ * How long past its wait a batched call's `started` line may take to arrive
+ * before the call is given up as never started.
+ */
+const BATCH_START_GRACE_MS = 2_000;
+
 /**
  * Header the dispatcher marks its own failures with. Anything carrying it never
  * reached a handler, so its status says nothing about the delivery.
@@ -92,6 +128,12 @@ const PLATFORM_ERROR_CODES = new Set([
  */
 export const EVENTS_DEPLOYED_HEADER = 'x-puter-events-deployed';
 
+/**
+ * Names the user whose scope a script belongs to, so a miss deploys it from
+ * their handlers. Absent for an app's own scope.
+ */
+export const EVENTS_HOLDER_HEADER = 'x-puter-events-holder';
+
 /**
  * Dispatch-error reasons this backend can resolve itself by deploying, rather
  * than wait on the rehydrate callback finding a backend behind the dispatcher's
@@ -112,21 +154,90 @@ const SELF_DEPLOYABLE_MISS_REASONS = new Set([
  */
 const ALREADY_DEPLOYED_MISS_REASONS = new Set(['deploy-timeout']);
 
+/** Whose call asked for the deploy, for the share of deploys it may take. */
+export interface WorkerMissOptions {
+    lane?: InvokeLane;
+    /**
+     * Holder uuid of the user scope the script belongs to; absent for app
+     * scope.
+     */
+    holder?: string;
+}
+
 /** What a self-deploy attempt did, and to which app/script it applies. */
 export type WorkerMissHandler = (
     appUid: string,
     script: string,
+    options: WorkerMissOptions,
 ) => Promise<'deployed' | string>;
 
+/**
+ * `work`'s result, or null once `ms` has passed. The work keeps running, and a
+ * failure after the cutoff is logged since nothing else would see it.
+ */
+const withinTime = async <T>(
+    work: Promise<T>,
+    ms: number,
+): Promise<T | null> => {
+    let timedOut = false;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const limit = new Promise<null>((resolve) => {
+        timer = setTimeout(
+            () => {
+                timedOut = true;
+                resolve(null);
+            },
+            Math.max(0, ms),
+        );
+    });
+    void (async () => {
+        try {
+            await work;
+        } catch (err) {
+            if (timedOut) console.warn('[events] deploy failed late', err);
+        }
+    })();
+    try {
+        return await Promise.race([work, limit]);
+    } finally {
+        clearTimeout(timer);
+    }
+};
+
 /** How long a handler has to answer before the attempt is abandoned. */
 export const EVENTS_INVOKE_TIMEOUT_MS = 30_000;
 
+/**
+ * How many invoke timeouts a delivery's worker lease lasts. The lease store
+ * reads this, and `invoke()` keeps a whole call, deploy and resend included,
+ * inside it.
+ */
+export const EVENTS_WORKER_LEASE_FACTOR = 2;
+
+/**
+ * Left unspent at the end of the lease, and the least a resend is worth
+ * starting with: below it the call is given up as `unavailable` and retried.
+ */
+export const EVENTS_DEADLINE_MARGIN_MS = 5_000;
+
 /**
  * Connections held open to the dispatcher, so a busy fleet is not
  * re-handshaking. Also how many calls one transport makes at once: a call past
  * that waits for a free one before its invoke timeout starts.
  */
 export const INVOKE_CONNECTIONS = 256;
+
+/**
+ * Of those, how many schedule runs may hold at once, so a burst of schedules
+ * can never take the connections event deliveries need.
+ */
+export const SCHEDULE_INVOKE_CONNECTIONS = 64;
+
+/** Whose call this is, for the connection share it may take. */
+export type InvokeLane = 'delivery' | 'schedule';
+
+/** What a deploy answers when another deploy of the region holds the way. */
+export const DEPLOY_BUSY = 'busy';
 const INVOKE_KEEP_ALIVE_MS = 30_000;
 
 /**
@@ -141,6 +252,8 @@ export interface WorkerInvokeRequest {
     /** Script the app's current handler set deploys as. */
     script: string;
     appUid: string;
+    /** Holder uuid, when the handler lives in a user scope. */
+    holder?: string;
     handler: string;
     /** The subscriber's app session token. */
     token: string;
@@ -154,6 +267,8 @@ export interface WorkerInvokeRequest {
 export interface EventsInvokeCall {
     script: string;
     appUid: string;
+    /** Holder uuid of the user scope the script belongs to. */
+    holder?: string;
     /** Value the script compares against its own invoke-key binding. */
     key: string;
     /** Serialized `{ handler, event, ctx, token }`. */
@@ -161,6 +276,12 @@ export interface EventsInvokeCall {
     timeoutMs: number;
     /** Set on the retry after this backend deployed the script itself. */
     deployed?: boolean;
+    lane?: InvokeLane;
+    /**
+     * Builds `body` again at send time, for a transport that waits for a
+     * connection first, so a send time stamped into it is the real one.
+     */
+    render?: () => string;
 }
 
 /** What one attempt at delivering an invocation came back with. */
@@ -176,6 +297,8 @@ export interface EventsInvokeAnswer {
     timedOut?: boolean;
     /** From a `retry-after` header in seconds. */
     retryAfterMs?: number;
+    /** A deploy this call needed was put off: another held the way. */
+    deployBusy?: boolean;
 }
 
 /**
@@ -198,6 +321,8 @@ export type WorkerInvokeOutcome =
 
 export interface WorkerInvokeResult {
     outcome: WorkerInvokeOutcome;
+    /** A deploy on miss was put off because another one held the way. */
+    deployBusy?: boolean;
     /** The status the handler answered with, or `null` when it never did. */
     status: number | null;
     /** Why it could not answer, when nothing did. */
@@ -206,6 +331,13 @@ export interface WorkerInvokeResult {
     retryAfterMs?: number;
 }
 
+/** How one invocation is sent. */
+export interface WorkerInvokeOptions {
+    lane?: InvokeLane;
+    /** Applied to the event as each attempt is sent. */
+    stampEvent?: (event: unknown) => unknown;
+}
+
 const NO_TRANSPORT =
     'no events dispatcher configured (url and internal secret)';
 
@@ -236,7 +368,10 @@ export class EventsWorkerInvokerClient extends PuterClient {
         this.#dispatcher = null;
     }
 
-    async invoke(request: WorkerInvokeRequest): Promise<WorkerInvokeResult> {
+    async invoke(
+        request: WorkerInvokeRequest,
+        options: WorkerInvokeOptions = {},
+    ): Promise<WorkerInvokeResult> {
         const transport = this.#pickTransport();
         if (!transport)
             return {
@@ -245,32 +380,80 @@ export class EventsWorkerInvokerClient extends PuterClient {
                 error: NO_TRANSPORT,
             };
 
-        const call: EventsInvokeCall = {
-            script: request.script,
-            appUid: request.appUid,
-            key: request.key,
-            body: JSON.stringify({
+        const { stampEvent } = options;
+        const render = (): string =>
+            JSON.stringify({
                 handler: request.handler,
-                event: request.event,
+                event: stampEvent ? stampEvent(request.event) : request.event,
                 ctx: request.ctx,
                 token: request.token,
-            }),
-            timeoutMs: this.#timeoutMs(),
-        };
+            });
+        const timeoutMs = this.#timeoutMs();
+        // The whole call, deploy and resend included, ends before the lease a
+        // delivery holds would let it be claimed again mid-run.
+        // A short configured timeout gets a proportionally short margin.
+        const margin = Math.min(EVENTS_DEADLINE_MARGIN_MS, timeoutMs / 2);
+        const deadline =
+            Date.now() + timeoutMs * EVENTS_WORKER_LEASE_FACTOR - margin;
+        const send = (
+            deployed: boolean,
+            sendTimeoutMs = timeoutMs,
+        ): Promise<EventsInvokeAnswer> =>
+            transport.send({
+                script: request.script,
+                appUid: request.appUid,
+                ...(request.holder ? { holder: request.holder } : {}),
+                key: request.key,
+                body: render(),
+                timeoutMs: sendTimeoutMs,
+                ...(deployed ? { deployed: true } : {}),
+                ...(options.lane ? { lane: options.lane } : {}),
+                ...(stampEvent ? { render } : {}),
+            });
 
-        let result = await transport.send(call);
-        if (this.#missHandler && selfDeployable(result)) {
-            const outcome = await this.#missHandler(
-                request.appUid,
-                request.script,
+        // A send may wait for a connection for half its timeout before the
+        // run's own timeout starts, so it is given what is left divided by that.
+        const resend = (): Promise<EventsInvokeAnswer> | null => {
+            const remaining = deadline - Date.now();
+            if (remaining < margin) return null;
+            return send(
+                true,
+                Math.floor(
+                    Math.min(
+                        timeoutMs,
+                        remaining / (1 + CONNECTION_WAIT_SHARE),
+                    ),
+                ),
             );
+        };
+        const outOfTime = (): WorkerInvokeResult => ({
+            outcome: 'unavailable',
+            status: null,
+            error: 'out of time to resend after deploy',
+        });
+
+        let result = await send(false);
+        if (this.#missHandler && selfDeployable(result)) {
+            // The deploy may be shared with other callers, so it is left
+            // running when this call runs out of time for it.
+            const outcome = await withinTime(
+                this.#missHandler(request.appUid, request.script, {
+                    lane: options.lane,
+                    ...(request.holder ? { holder: request.holder } : {}),
+                }),
+                deadline - Date.now() - margin,
+            );
+            if (outcome === null) return outOfTime();
             if (outcome !== 'deployed')
                 return {
                     outcome: 'unavailable',
                     status: null,
                     error: `deploy: ${outcome}`,
+                    ...(outcome === DEPLOY_BUSY ? { deployBusy: true } : {}),
                 };
-            result = await transport.send({ ...call, deployed: true });
+            const resent = await resend();
+            if (!resent) return outOfTime();
+            result = resent;
         } else if (
             result.status === null &&
             result.dispatchReason &&
@@ -278,10 +461,19 @@ export class EventsWorkerInvokerClient extends PuterClient {
         ) {
             // No deploy to trigger — just give propagation the one retry
             // it usually needs, without spending an upload on it.
-            result = await transport.send({ ...call, deployed: true });
+            const resent = await resend();
+            if (!resent) return outOfTime();
+            result = resent;
         }
 
         const { status, handled, error, timedOut, retryAfterMs } = result;
+        if (result.deployBusy)
+            return {
+                outcome: 'unavailable',
+                status: null,
+                error,
+                deployBusy: true,
+            };
         // Past the timeout the handler was most likely still running, which
         // is its own doing; any other missing answer never reached it.
         if (status === null)
@@ -313,9 +505,19 @@ export class EventsWorkerInvokerClient extends PuterClient {
         // a misconfiguration worth reporting once per attempt rather than a
         // round trip.
         if (!dispatcherUrl || !internalSecret) return null;
+        const { invokeBatchSize, invokeBatchWindowMs } =
+            this.config.events ?? {};
         this.#dispatcher ??= new DispatcherInvokeTransport(
             dispatcherUrl,
             internalSecret,
+            {
+                ...(invokeBatchSize !== undefined
+                    ? { batchSize: invokeBatchSize }
+                    : {}),
+                ...(invokeBatchWindowMs !== undefined
+                    ? { batchWindowMs: invokeBatchWindowMs }
+                    : {}),
+            },
         );
         return this.#dispatcher;
     }
@@ -328,28 +530,63 @@ export class EventsWorkerInvokerClient extends PuterClient {
     }
 }
 
+/** A call waiting in, or sent with, the current batch. */
+interface BatchedCall {
+    call: EventsInvokeCall;
+    resolve: (answer: EventsInvokeAnswer) => void;
+    /** When its wait for a connection began, for what is left of it. */
+    waitingSince: number;
+}
+
+export interface DispatcherInvokeTransportOptions {
+    fetchImpl?: typeof undiciFetch;
+    /** Calls per batch; 1 sends every call on its own. */
+    batchSize?: number;
+    batchWindowMs?: number;
+}
+
 /** Reaches an events worker the only way production can: through the dispatcher. */
 export class DispatcherInvokeTransport implements EventsInvokeTransport {
     readonly #url: string;
     readonly #secret: string;
     readonly #fetchImpl: typeof undiciFetch;
+    readonly #batchSize: number;
+    readonly #batchWindowMs: number;
     #agent: Agent | null = null;
     /** Calls holding a connection slot. */
     #active = 0;
-    /** Calls waiting for one, longest-waiting first. */
-    readonly #waiting = new Set<() => void>();
+    /** Of those, schedule runs. */
+    #scheduleActive = 0;
+    /** Calls waiting for one, longest-waiting first, with their lane. */
+    readonly #waiting = new Map<() => void, InvokeLane>();
+    /** Calls gathered for the next batch. */
+    #batch: BatchedCall[] = [];
+    #batchTimer: ReturnType<typeof setTimeout> | null = null;
+    /** Until when the dispatcher is taken not to have the batch route. */
+    #batchUnsupportedUntil = 0;
 
     constructor(
         dispatcherUrl: string,
         secret: string,
-        options: { fetchImpl?: typeof undiciFetch } = {},
+        options: DispatcherInvokeTransportOptions = {},
     ) {
-        this.#url = dispatcherUrl;
+        // Joined as a string: `new URL(path, base)` resolves an absolute path
+        // against the base's origin, dropping any path prefix it carries.
+        this.#url = dispatcherUrl.replace(/\/$/, '');
         this.#secret = secret;
         this.#fetchImpl = options.fetchImpl ?? undiciFetch;
+        this.#batchSize = Math.max(
+            1,
+            Math.floor(options.batchSize ?? INVOKE_BATCH_SIZE),
+        );
+        this.#batchWindowMs = Math.max(
+            0,
+            options.batchWindowMs ?? INVOKE_BATCH_WINDOW_MS,
+        );
     }
 
     close(): void {
+        this.#flush();
         void this.#agent?.close().catch(() => {});
         this.#agent = null;
     }
@@ -357,29 +594,71 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
     async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
         // No `timedOut`: the handler never got this call, so the wait is not
         // its failure.
-        if (!(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE)))
+        const lane = call.lane ?? 'delivery';
+        const waitingSince = Date.now();
+        if (
+            !(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE, lane))
+        )
             return {
                 status: null,
                 error: 'no free connection to the events dispatcher',
             };
         try {
-            // `new URL(path, base)` resolves an absolute path against the
-            // base's origin, dropping any path prefix `base` already carries.
-            const url = `${this.#url.replace(/\/$/, '')}${EVENTS_DISPATCH_PATH}`;
-            const response = await this.#fetchImpl(url, {
-                method: 'POST',
-                headers: {
-                    'content-type': 'application/json',
-                    'x-puter-internal-auth': this.#secret,
-                    'x-puter-events-script': call.script,
-                    'x-puter-events-app': call.appUid,
-                    'x-puter-events-key': call.key,
-                    ...(call.deployed ? { [EVENTS_DEPLOYED_HEADER]: '1' } : {}),
-                },
-                body: call.body,
-                signal: AbortSignal.timeout(call.timeoutMs),
-                dispatcher: this.#dispatcherFor(),
+            if (this.#batchSize === 1) return await this.#sendOne(call);
+            return await new Promise<EventsInvokeAnswer>((resolve) => {
+                this.#batch.push({ call, resolve, waitingSince });
+                if (this.#batch.length >= this.#batchSize) this.#flush();
+                else
+                    this.#batchTimer ??= setTimeout(
+                        () => this.#flush(),
+                        this.#batchWindowMs,
+                    );
             });
+        } finally {
+            this.#release(lane);
+        }
+    }
+
+    /** Send what has gathered: alone when it is one call, else as a batch. */
+    #flush(): void {
+        if (this.#batchTimer) clearTimeout(this.#batchTimer);
+        this.#batchTimer = null;
+        const batch = this.#batch;
+        this.#batch = [];
+        if (batch.length === 0) return;
+        if (batch.length === 1 || Date.now() < this.#batchUnsupportedUntil) {
+            for (const { call, resolve } of batch)
+                void this.#sendOne(call).then(resolve);
+            return;
+        }
+        void this.#sendBatch(batch);
+    }
+
+    /** One call on the single route. */
+    async #sendOne(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                        'x-puter-events-script': call.script,
+                        'x-puter-events-app': call.appUid,
+                        ...(call.holder
+                            ? { [EVENTS_HOLDER_HEADER]: call.holder }
+                            : {}),
+                        'x-puter-events-key': call.key,
+                        ...(call.deployed
+                            ? { [EVENTS_DEPLOYED_HEADER]: '1' }
+                            : {}),
+                    },
+                    body: call.render ? call.render() : call.body,
+                    signal: AbortSignal.timeout(call.timeoutMs),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
             // The body is not read: a handler answers with a status, and
             // whatever else it writes is its own business.
             void response.body?.cancel().catch(() => {});
@@ -390,24 +669,182 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
                 EVENTS_DISPATCH_ERROR_HEADER,
             );
             if (dispatchError !== null)
-                return {
-                    status: null,
-                    error: `dispatcher: ${dispatchError} (${response.status})`,
-                    dispatchReason: dispatchError,
-                };
+                return dispatchAnswer(dispatchError, response.status);
 
             return answerFrom(response);
         } catch (err) {
             return failedAnswer(err);
-        } finally {
-            this.#release();
         }
     }
 
+    /**
+     * Several calls in one request. A call may wait in the dispatcher's queue
+     * for whatever is left of its connection wait, and its timeout runs from
+     * its `started` line; the request itself is dropped once every call has an
+     * answer.
+     */
+    async #sendBatch(batch: BatchedCall[]): Promise<void> {
+        const hangUp = new AbortController();
+        let open = batch.length;
+        const answered = batch.map(() => false);
+        const timers: ReturnType<typeof setTimeout>[] = [];
+        const settle = (i: number, answer: EventsInvokeAnswer): void => {
+            if (answered[i]) return;
+            answered[i] = true;
+            clearTimeout(timers[i]);
+            batch[i].resolve(answer);
+            if (--open === 0) hangUp.abort();
+        };
+        const settleRest = (answer: EventsInvokeAnswer): void => {
+            for (let i = 0; i < batch.length; i++) settle(i, answer);
+        };
+
+        const now = Date.now();
+        const waits = batch.map(({ call, waitingSince }) =>
+            Math.max(
+                0,
+                Math.floor(
+                    call.timeoutMs * CONNECTION_WAIT_SHARE -
+                        (now - waitingSince),
+                ),
+            ),
+        );
+        const body = JSON.stringify({
+            calls: batch.map(({ call }, i) => ({
+                script: call.script,
+                appUid: call.appUid,
+                ...(call.holder ? { holder: call.holder } : {}),
+                key: call.key,
+                ...(call.deployed ? { deployed: true } : {}),
+                body: call.render ? call.render() : call.body,
+                waitMs: waits[i],
+            })),
+        });
+        const arm = (
+            i: number,
+            ms: number,
+            answer: () => EventsInvokeAnswer,
+        ) => {
+            clearTimeout(timers[i]);
+            timers[i] = setTimeout(() => settle(i, answer()), ms);
+            timers[i].unref?.();
+        };
+        // No `timedOut`: a call that never started never reached a handler.
+        batch.forEach((_, i) =>
+            arm(i, waits[i] + BATCH_START_GRACE_MS, () => ({
+                status: null,
+                error: 'dispatcher: call never started',
+            })),
+        );
+        const started = (i: number): void => {
+            if (!answered[i])
+                arm(i, batch[i].call.timeoutMs, () =>
+                    failedAnswer(timeoutError()),
+                );
+        };
+        const longest = Math.max(
+            ...batch.map(
+                ({ call }, i) =>
+                    waits[i] + BATCH_START_GRACE_MS + call.timeoutMs,
+            ),
+        );
+
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_BATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                    },
+                    body,
+                    signal: AbortSignal.any([
+                        hangUp.signal,
+                        AbortSignal.timeout(longest),
+                    ]),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
+
+            const dispatchError = response.headers.get(
+                EVENTS_DISPATCH_ERROR_HEADER,
+            );
+            if (dispatchError === 'no-such-route') {
+                // A dispatcher from before the batch route: nothing ran, so
+                // every call goes again on its own.
+                void response.body?.cancel().catch(() => {});
+                this.#batchUnsupportedUntil =
+                    Date.now() + BATCH_UNSUPPORTED_RETRY_MS;
+                batch.forEach(({ call }, i) => {
+                    if (!answered[i])
+                        void this.#sendOne(call).then((a) => settle(i, a));
+                });
+                return;
+            }
+            if (dispatchError !== null || !response.ok) {
+                void response.body?.cancel().catch(() => {});
+                // An unmarked failure (an edge 404, a WAF 403) reads as
+                // Puter's, never as any handler's answer.
+                settleRest(
+                    dispatchError !== null
+                        ? dispatchAnswer(dispatchError, response.status)
+                        : {
+                              status: null,
+                              error: `dispatcher: unmarked ${response.status} for a batch`,
+                          },
+                );
+                return;
+            }
+
+            let buffered = '';
+            const read = (line: string): void => {
+                if (line.trim() === '') return;
+                const parsed = parseAnswerLine(line);
+                if (!parsed || parsed.i >= batch.length) return;
+                if ('started' in parsed) started(parsed.i);
+                else settle(parsed.i, parsed.answer);
+            };
+            const decoder = new TextDecoder();
+            for await (const chunk of response.body ?? []) {
+                buffered += decoder.decode(chunk, { stream: true });
+                let newline = buffered.indexOf('\n');
+                while (newline !== -1) {
+                    read(buffered.slice(0, newline));
+                    buffered = buffered.slice(newline + 1);
+                    newline = buffered.indexOf('\n');
+                }
+            }
+            read(buffered);
+            settleRest({
+                status: null,
+                error: 'dispatcher: batch ended without an answer',
+            });
+        } catch (err) {
+            // Every call has its answer and the request was dropped on purpose.
+            if (open === 0) return;
+            settleRest(failedAnswer(err));
+        }
+    }
+
+    /** Whether a call on `lane` may take a connection now. */
+    #free(lane: InvokeLane): boolean {
+        return (
+            this.#active < INVOKE_CONNECTIONS &&
+            (lane !== 'schedule' ||
+                this.#scheduleActive < SCHEDULE_INVOKE_CONNECTIONS)
+        );
+    }
+
+    #take(lane: InvokeLane): void {
+        this.#active++;
+        if (lane === 'schedule') this.#scheduleActive++;
+    }
+
     /** A connection slot, or false once `waitMs` passes without one. */
-    #acquire(waitMs: number): Promise<boolean> {
-        if (this.#active < INVOKE_CONNECTIONS) {
-            this.#active++;
+    #acquire(waitMs: number, lane: InvokeLane): Promise<boolean> {
+        if (this.#free(lane)) {
+            this.#take(lane);
             return Promise.resolve(true);
         }
         return new Promise((resolve) => {
@@ -420,19 +857,24 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
                 resolve(false);
             }, waitMs);
             timer.unref?.();
-            this.#waiting.add(granted);
+            this.#waiting.set(granted, lane);
         });
     }
 
-    /** Hand the slot to the longest waiter, or give it back. */
-    #release(): void {
-        const next = this.#waiting.values().next().value;
-        if (next) {
-            this.#waiting.delete(next);
-            next();
+    /**
+     * Give the slot back, then hand one to the longest waiter it can go to: a
+     * schedule waiter at a full schedule share doesn't hold up a delivery.
+     */
+    #release(lane: InvokeLane): void {
+        this.#active--;
+        if (lane === 'schedule') this.#scheduleActive--;
+        for (const [granted, waiting] of this.#waiting) {
+            if (!this.#free(waiting)) continue;
+            this.#waiting.delete(granted);
+            this.#take(waiting);
+            granted();
             return;
         }
-        this.#active--;
     }
 
     /** One pool per process, built on first use and closed with the server. */
@@ -446,6 +888,65 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
     }
 }
 
+/** What a failure the dispatcher marked as its own reads as. */
+const dispatchAnswer = (
+    reason: string,
+    status: number,
+): EventsInvokeAnswer => ({
+    status: null,
+    error: `dispatcher: ${reason} (${status})`,
+    dispatchReason: reason,
+});
+
+/** What `AbortSignal.timeout` rejects with, for a batched call's own timeout. */
+const timeoutError = (): Error =>
+    new DOMException(
+        'The operation was aborted due to timeout',
+        'TimeoutError',
+    );
+
+/** One line of a batch answer, or null when it is not one. */
+export const parseAnswerLine = (
+    line: string,
+):
+    | { i: number; started: true }
+    | { i: number; answer: EventsInvokeAnswer }
+    | null => {
+    let raw: unknown;
+    try {
+        raw = JSON.parse(line);
+    } catch {
+        return null;
+    }
+    if (!raw || typeof raw !== 'object') return null;
+    const { i, started, status, handled, error, retryAfter, dispatch } =
+        raw as Record<string, unknown>;
+    if (!Number.isInteger(i) || (i as number) < 0) return null;
+    if (started === true) return { i: i as number, started: true };
+    if (typeof status !== 'number') return null;
+    if (typeof dispatch === 'string')
+        return { i: i as number, answer: dispatchAnswer(dispatch, status) };
+    return {
+        i: i as number,
+        answer: answerFrom({
+            status,
+            headers: {
+                get: (name) => {
+                    if (name === EVENTS_HANDLED_HEADER)
+                        return handled === true ? '1' : null;
+                    if (name === EVENTS_ERROR_HEADER)
+                        return typeof error === 'string' ? error : null;
+                    if (name === 'retry-after')
+                        return typeof retryAfter === 'string'
+                            ? retryAfter
+                            : null;
+                    return null;
+                },
+            },
+        }),
+    };
+};
+
 /** Whether deploying the script is what fixes this answer. */
 const selfDeployable = (result: EventsInvokeAnswer): boolean =>
     result.status === null

src/backend/controllers/auth/AuthController.test.ts

@@ -1416,7 +1416,6 @@ describe('AuthController.handleLogin', () => {
             is_user_token: true,
         });
         expect(body.user.directories).toBeTypeOf('object');
-        expect(Array.isArray(body.user.taskbar_items)).toBe(true);
     });
 
     // A deleted row still answering from cache reaches the session INSERT.

src/backend/controllers/broadcast/BroadcastController.ts

@@ -19,6 +19,7 @@
 
 import type { Request, Response } from 'express';
 import { Controller, Post } from '../../core/http/decorators.js';
+import { schedulesUnavailable } from '../../core/schedules.js';
 import type { BroadcastService } from '../../services/broadcast/BroadcastService.js';
 import type { ForwardBatch } from '../../services/events/forwardQueue.js';
 import { PuterController } from '../types.js';
@@ -108,6 +109,42 @@ export class BroadcastController extends PuterController {
         );
         res.status(200).json({ ok: true, ...reply });
     }
+
+    /**
+     * Schedule changes a region made for schedules this one owns, handed to
+     * `services.schedules`; `events_disabled` on a server without it.
+     */
+    @Post('/events/schedules', { subdomain: '*' })
+    async schedules(req: Request, res: Response): Promise<void> {
+        const broadcast = this.services.broadcast as unknown as
+            BroadcastService | undefined;
+        if (!broadcast) {
+            res.status(503).json({
+                error: { message: 'Broadcast service not registered' },
+            });
+            return;
+        }
+
+        const verified = await broadcast.verifySignedRequest(
+            req.rawBody,
+            broadcastHeaders(req),
+        );
+        if (!verified.ok) {
+            res.status(verified.status ?? 400).json({
+                error: { message: verified.message ?? 'Bad request' },
+            });
+            return;
+        }
+        if (verified.info?.ignored) {
+            res.status(200).json({ ok: true, ...verified.info });
+            return;
+        }
+
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        const reply = await schedules.receivePeerChanges(req.body);
+        res.status(200).json({ ok: true, ...reply });
+    }
 }
 
 const broadcastHeaders = (req: Request) => {

src/backend/controllers/desktop/DesktopController.js

@@ -25,18 +25,13 @@ const ALLOWED_SORT_BY = ['name', 'size', 'modified', 'type'];
 const ALLOWED_SORT_ORDER = ['asc', 'desc'];
 
 /**
- * Desktop/UI preference routes.
- *
- * Two categories:
- *
- * - User-level: desktop background, taskbar items (UserStore)
- * - Folder-level: layout, sort_by/sort_order (fsentries table)
+ * Folder view preference routes: an explorer window's layout and sort order,
+ * kept on the folder's fsentry.
  */
 /**
- * Desktop preference writes — background, taskbar, layout, sort order. All four
- * persist to the user row on every call, and the GUI fires them on direct user
- * action, so a per-minute ceiling well above human speed is enough to catch a
- * stuck client.
+ * Folder preference writes — layout, sort order. The GUI fires them on direct
+ * user action, so a per-minute ceiling well above human speed is enough to
+ * catch a stuck client.
  */
 const PREFERENCE_WRITE_LIMIT = {
     scope: 'desktop-preference',
@@ -47,89 +42,6 @@ const PREFERENCE_WRITE_LIMIT = {
 
 export class DesktopController extends PuterController {
     registerRoutes(router) {
-        // -- Desktop background --------------------------------------
-
-        router.post(
-            '/set-desktop-bg',
-            {
-                subdomain: 'api',
-                requireUserActor: true,
-                allowFullAccessToken: true,
-                rateLimit: PREFERENCE_WRITE_LIMIT,
-            },
-            async (req, res) => {
-                const { url, color, fit } = req.body ?? {};
-
-                const patch = {};
-                if (url !== undefined) {
-                    if (url !== null && typeof url !== 'string') {
-                        throw new HttpError(
-                            400,
-                            '`url` must be a string or null',
-                            { legacyCode: 'bad_request' },
-                        );
-                    }
-                    patch.desktop_bg_url = url;
-                }
-                if (color !== undefined) {
-                    if (color !== null && typeof color !== 'string') {
-                        throw new HttpError(
-                            400,
-                            '`color` must be a string or null',
-                            { legacyCode: 'bad_request' },
-                        );
-                    }
-                    patch.desktop_bg_color = color;
-                }
-                if (fit !== undefined) {
-                    if (fit !== null && typeof fit !== 'string') {
-                        throw new HttpError(
-                            400,
-                            '`fit` must be a string or null',
-                            { legacyCode: 'bad_request' },
-                        );
-                    }
-                    patch.desktop_bg_fit = fit;
-                }
-
-                if (Object.keys(patch).length === 0) {
-                    throw new HttpError(400, 'No fields provided', {
-                        legacyCode: 'bad_request',
-                    });
-                }
-
-                await this.stores.user.update(req.actor.user.id, patch);
-                res.json({});
-            },
-        );
-
-        // -- Taskbar items -------------------------------------------
-
-        router.post(
-            '/update-taskbar-items',
-            {
-                subdomain: 'api',
-                requireUserActor: true,
-                allowFullAccessToken: true,
-                rateLimit: PREFERENCE_WRITE_LIMIT,
-            },
-            async (req, res) => {
-                const { items } = req.body ?? {};
-                if (!Array.isArray(items)) {
-                    throw new HttpError(
-                        400,
-                        'Missing or invalid `items` array',
-                        { legacyCode: 'bad_request' },
-                    );
-                }
-
-                await this.stores.user.update(req.actor.user.id, {
-                    taskbar_items: JSON.stringify(items),
-                });
-                res.json({});
-            },
-        );
-
         // -- Folder layout -------------------------------------------
 
         router.post(

src/backend/controllers/desktop/DesktopController.test.ts

@@ -130,150 +130,6 @@ const callRoute = async (
     });
 };
 
-// ── /set-desktop-bg ─────────────────────────────────────────────────
-
-describe('DesktopController POST /set-desktop-bg', () => {
-    it('persists url/color/fit on the user row', async () => {
-        const { actor, userId } = await makeUser();
-        const { res, captured } = makeRes();
-        await callRoute(
-            'post',
-            '/set-desktop-bg',
-            makeReq({
-                body: {
-                    url: 'https://cdn.test/wallpaper.png',
-                    color: '#000000',
-                    fit: 'cover',
-                },
-                actor,
-            }),
-            res,
-        );
-        expect(captured.body).toEqual({});
-
-        const refreshed = await server.stores.user.getById(userId);
-        expect(refreshed?.desktop_bg_url).toBe(
-            'https://cdn.test/wallpaper.png',
-        );
-        expect(refreshed?.desktop_bg_color).toBe('#000000');
-        expect(refreshed?.desktop_bg_fit).toBe('cover');
-    });
-
-    it('persists only the supplied fields (partial update)', async () => {
-        const { actor, userId } = await makeUser();
-        const { res } = makeRes();
-        await callRoute(
-            'post',
-            '/set-desktop-bg',
-            makeReq({ body: { color: '#ffffff' }, actor }),
-            res,
-        );
-        const refreshed = await server.stores.user.getById(userId);
-        expect(refreshed?.desktop_bg_color).toBe('#ffffff');
-        // Untouched fields remain at their column default.
-        expect(refreshed?.desktop_bg_url).toBeFalsy();
-        expect(refreshed?.desktop_bg_fit).toBeFalsy();
-    });
-
-    it('passes through `null` to clear a field', async () => {
-        const { actor, userId } = await makeUser();
-        // First populate, then clear.
-        const populate = makeRes();
-        await callRoute(
-            'post',
-            '/set-desktop-bg',
-            makeReq({
-                body: { url: 'https://cdn.test/x.png' },
-                actor,
-            }),
-            populate.res,
-        );
-        const before = await server.stores.user.getById(userId);
-        expect(before?.desktop_bg_url).toBe('https://cdn.test/x.png');
-
-        const clear = makeRes();
-        await callRoute(
-            'post',
-            '/set-desktop-bg',
-            makeReq({ body: { url: null }, actor }),
-            clear.res,
-        );
-        const after = await server.stores.user.getById(userId);
-        expect(after?.desktop_bg_url).toBeNull();
-    });
-
-    it('throws 400 when url is not a string or null', async () => {
-        const { actor } = await makeUser();
-        const { res } = makeRes();
-        await expect(
-            callRoute(
-                'post',
-                '/set-desktop-bg',
-                makeReq({ body: { url: 123 }, actor }),
-                res,
-            ),
-        ).rejects.toMatchObject({ statusCode: 400 });
-    });
-
-    it('throws 400 when no fields are provided', async () => {
-        const { actor } = await makeUser();
-        const { res } = makeRes();
-        await expect(
-            callRoute(
-                'post',
-                '/set-desktop-bg',
-                makeReq({ body: {}, actor }),
-                res,
-            ),
-        ).rejects.toMatchObject({ statusCode: 400 });
-    });
-});
-
-// ── /update-taskbar-items ───────────────────────────────────────────
-
-describe('DesktopController POST /update-taskbar-items', () => {
-    it('persists items as a JSON-encoded string', async () => {
-        const { actor, userId } = await makeUser();
-        const items = [{ name: 'editor' }, { name: 'browser' }];
-        const { res } = makeRes();
-        await callRoute(
-            'post',
-            '/update-taskbar-items',
-            makeReq({ body: { items }, actor }),
-            res,
-        );
-
-        const refreshed = await server.stores.user.getById(userId);
-        expect(refreshed?.taskbar_items).toBe(JSON.stringify(items));
-    });
-
-    it('throws 400 when items is missing', async () => {
-        const { actor } = await makeUser();
-        const { res } = makeRes();
-        await expect(
-            callRoute(
-                'post',
-                '/update-taskbar-items',
-                makeReq({ body: {}, actor }),
-                res,
-            ),
-        ).rejects.toMatchObject({ statusCode: 400 });
-    });
-
-    it('throws 400 when items is not an array', async () => {
-        const { actor } = await makeUser();
-        const { res } = makeRes();
-        await expect(
-            callRoute(
-                'post',
-                '/update-taskbar-items',
-                makeReq({ body: { items: 'oops' }, actor }),
-                res,
-            ),
-        ).rejects.toMatchObject({ statusCode: 400 });
-    });
-});
-
 // ── /set_layout ─────────────────────────────────────────────────────
 
 describe('DesktopController POST /set_layout', () => {

src/backend/controllers/events/EventsController.ts

@@ -25,6 +25,11 @@ import { bodyRecord } from '../../core/http/requestBody.js';
 import { DURABLE_LIST_LIMIT_CAP } from '../../stores/events/DurableSubscriptionStore.js';
 import { EVENTS_WORKERS_LIST_LIMIT_CAP } from '../../stores/events/EventHandlerStore.js';
 import { KV_HANDLE_LIST_LIMIT_CAP } from '../../stores/events/KvShareHandleStore.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    schedulesUnavailable,
+    type ScheduleServiceContract,
+} from '../../core/schedules.js';
 import { normalizeLimit } from '../../util/pagination.js';
 import { PuterController } from '../types.js';
 import { EVENTS_WORKER_PREFIX } from '../../services/events/workerRuntime.js';
@@ -59,6 +64,8 @@ const EVENTS_SCRIPT_REGEX = new RegExp(`^${EVENTS_WORKER_PREFIX}[a-f0-9]{32}$`);
 
 /** Matches the edge dispatcher's own validation of the app uid it forwards. */
 const APP_UID_REGEX = /^app-[A-Za-z0-9_-]{1,64}$/;
+/** A user uuid. */
+const HOLDER_REGEX = /^[A-Za-z0-9-]{1,64}$/;
 
 @Controller('/events')
 export class EventsController extends PuterController {
@@ -127,7 +134,8 @@ export class EventsController extends PuterController {
      *
      * Stateless: no subscription, no stored position, nothing written. The
      * cursor comes back to the caller and goes out again as `cursor` (or
-     * `after`, the name it first shipped under).
+     * `after`, the name it first shipped under). `from=end` with no cursor
+     * starts at the newest row instead of the oldest.
      */
     @Get('/fetch', {
         subdomain: 'api',
@@ -138,9 +146,16 @@ export class EventsController extends PuterController {
     async fetch(req: Request, res: Response): Promise<void> {
         const actor = this.#requireActor(req);
         const query = (req.query ?? {}) as Record<string, unknown>;
+        const from = query.from as 'start' | 'end' | undefined;
+        if (from !== undefined && from !== 'start' && from !== 'end') {
+            throw new HttpError(400, '`from` must be `start` or `end`', {
+                legacyCode: 'bad_request',
+            });
+        }
 
         const page = await this.services.events.fetchMissed(actor, {
             subject: typeof query.subject === 'string' ? query.subject : '',
+            from,
             cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
             after: typeof query.after === 'string' ? query.after : undefined,
             limit: normalizeLimit(query.limit, {
@@ -169,6 +184,88 @@ export class EventsController extends PuterController {
         res.json({});
     }
 
+    // -- Schedules ---------------------------------------------------
+    //
+    // An app's handler run on a timer, served by `services.schedules` when a
+    // server has it and answered `events_disabled` when it doesn't. Creating
+    // one needs an app context; listing, reading and removing are scoped the
+    // way the durable verbs are. The create and remove verbs spend the
+    // subscribe and unsubscribe budgets in the service, shared with those
+    // verbs.
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /** POST /events/schedules — create a schedule for the calling app. */
+    @Post('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async createSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().createFor(actor, bodyRecord(req)));
+    }
+
+    /** GET /events/schedules — the caller's schedules, one page at a time. */
+    @Get('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async listSchedules(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        const query = (req.query ?? {}) as Record<string, unknown>;
+
+        const page = await this.#schedules().list(actor, {
+            limit: normalizeLimit(query.limit, {
+                cap: SCHEDULE_LIST_LIMIT_CAP,
+            }),
+            cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
+            includeTotal: query.includeTotal === 'true',
+        });
+
+        res.json({
+            items: page.items,
+            ...(page.cursor ? { cursor: page.cursor } : {}),
+            ...(page.total !== undefined ? { total: page.total } : {}),
+        });
+    }
+
+    /**
+     * POST /events/schedules/remove — end one schedule. An id the caller does
+     * not hold reads as absent.
+     */
+    @Post('/schedules/remove', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async removeSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        await this.#schedules().remove(actor, bodyRecord(req).scheduleId);
+        res.json({});
+    }
+
+    /**
+     * GET /events/schedules/:scheduleId — one schedule, ended or not. An id the
+     * caller does not hold reads as absent.
+     */
+    @Get('/schedules/:scheduleId', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async getSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().get(actor, req.params.scheduleId));
+    }
+
     // -- Cross-user key-value handles --------------------------------
 
     /**
@@ -315,6 +412,7 @@ export class EventsController extends PuterController {
             handlers: await this.services.events.listHandlers(actor, {
                 appUid:
                     typeof query.appUid === 'string' ? query.appUid : undefined,
+                scope: query.scope,
             }),
         });
     }
@@ -413,6 +511,15 @@ export class EventsController extends PuterController {
             throw new HttpError(400, 'Missing or invalid `script`/`appUid`', {
                 legacyCode: 'bad_request',
             });
+        // Absent or empty is the app's own scope.
+        const holder = body.holder === undefined ? '' : body.holder;
+        if (
+            typeof holder !== 'string' ||
+            !(holder === '' || HOLDER_REGEX.test(holder))
+        )
+            throw new HttpError(400, 'Invalid `holder`', {
+                legacyCode: 'bad_request',
+            });
 
         const deployer = this.#eventsWorkerDeployer();
         if (!deployer.enabled) {
@@ -420,15 +527,25 @@ export class EventsController extends PuterController {
             return;
         }
 
-        const outcome = await deployer.ensure(appUid, script);
+        const scope = await deployer.scopeFor(appUid, holder);
+        if (!scope) {
+            res.status(404).json({ deployed: false, reason: 'no-holder' });
+            return;
+        }
+
+        const outcome = await deployer.ensure(scope, script);
         if (outcome === 'deployed') {
             res.json({ deployed: true });
             return;
         }
-        // `failed` and `throttled` are worth another attempt: the rest say
-        // this script is not what the app's handlers currently are.
+        // `failed`, `throttled` and `busy` are worth another attempt: the rest
+        // say this script is not what the app's handlers currently are.
         res.status(
-            outcome === 'failed' || outcome === 'throttled' ? 502 : 404,
+            outcome === 'failed' ||
+                outcome === 'throttled' ||
+                outcome === 'busy'
+                ? 502
+                : 404,
         ).json({
             deployed: false,
             reason: outcome,
@@ -449,14 +566,22 @@ export class EventsController extends PuterController {
     override onServerStart(): void {
         const deployer = this.#eventsWorkerDeployer();
         if (!deployer.enabled) return;
+        this.services.events.useEagerDeploy((scope, script) =>
+            deployer.deployEagerly(scope, script),
+        );
         if (this.config.workers?.localServer) {
             this.services.events.useWorkerTransport(
                 new LocalEventsInvokeTransport(this.services, deployer),
             );
             return;
         }
-        this.services.events.useWorkerMissHandler((appUid, script) =>
-            deployer.ensure(appUid, script),
+        this.services.events.useWorkerMissHandler(
+            async (appUid, script, options) => {
+                const scope = await deployer.scopeFor(appUid, options.holder);
+                return scope
+                    ? deployer.ensure(scope, script, options)
+                    : 'no-holder';
+            },
         );
     }
 

src/backend/controllers/events/limits.test.ts

@@ -34,23 +34,46 @@ import {
     EVENTS_DURABLE_SUBSCRIPTIONS_MAX,
     EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP,
     EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
+    EVENTS_HANDLER_PUBLISH_LIMIT,
+    EVENTS_HANDLER_SOURCE_MAX_BYTES,
+    EVENTS_HANDLERS_PER_APP,
     EVENTS_KV_HANDLES_PER_APP,
     EVENTS_KV_HANDLES_PER_USER,
     EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT,
     EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT,
     EVENTS_SINGLE_DELIVERY_LIMIT,
+    EVENTS_USER_HANDLERS_PER_SCOPE,
+    EVENTS_USER_SCOPE_SOURCE_MAX_BYTES,
+    EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
+    EVENTS_WORKER_DEPLOYS_PER_HOUR,
     EVENTS_WORKER_INVOCATION_LIMIT,
+    EVENTS_WORKER_SOURCE_MAX_BYTES,
+    RENTED_UNITS_PER_USER,
     limitFor,
+    tiered,
     type TieredLimit,
 } from './limits.js';
 
 const tiers: Array<[string, TieredLimit]> = [
-    ['durable subscriptions per account', EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER],
+    [
+        'durable subscriptions per account',
+        EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
+    ],
     ['durable subscriptions per app', EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP],
     ['live share handles per account', EVENTS_KV_HANDLES_PER_USER],
     ['live share handles per app', EVENTS_KV_HANDLES_PER_APP],
 ];
 
+describe('tiered', () => {
+    it('gives paid plans the base, and the free and temporary tiers their own', () => {
+        const limit = tiered(30, 20, 10);
+        expect(limitFor(limit, null)).toBe(30);
+        expect(limitFor(limit, 'some_plan')).toBe(30);
+        expect(limitFor(limit, DEFAULT_FREE_SUBSCRIPTION)).toBe(20);
+        expect(limitFor(limit, DEFAULT_TEMP_SUBSCRIPTION)).toBe(10);
+    });
+});
+
 describe('the tiered subscription quotas', () => {
     it.each(tiers)('%s never lets a free tier exceed paid', (_name, tier) => {
         for (const n of Object.values(tier.bySubscription))
@@ -105,9 +128,9 @@ describe('the tiered subscription quotas', () => {
     });
 
     it('holds an unrecognised plan to the paid base', () => {
-        expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan')).toBe(
-            EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
-        );
+        expect(
+            limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan'),
+        ).toBe(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit);
         expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, null)).toBe(
             EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
         );
@@ -131,9 +154,9 @@ describe('the delivery budgets', () => {
             expect(
                 limitFor(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT, plan),
             ).toBe(matched);
-            expect(
-                limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan),
-            ).toBe(evaluated);
+            expect(limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan)).toBe(
+                evaluated,
+            );
             expect(evaluated).toBeGreaterThanOrEqual(matched);
         }
     });
@@ -169,3 +192,42 @@ describe('the delivery budgets', () => {
         }
     });
 });
+
+describe('the handler limits', () => {
+    it('pins the app-scope numbers the quotas page lists', () => {
+        expect(EVENTS_HANDLERS_PER_APP).toBe(100);
+        expect(EVENTS_HANDLER_SOURCE_MAX_BYTES).toBe(64 * 1024);
+        expect(EVENTS_WORKER_SOURCE_MAX_BYTES).toBe(5 * 1024 * 1024);
+        expect(EVENTS_WORKER_DEPLOYS_PER_HOUR).toBe(30);
+        expect(EVENTS_HANDLER_PUBLISH_LIMIT.limit).toBe(60);
+        expect(EVENTS_HANDLER_PUBLISH_LIMIT.window).toBe(60_000);
+    });
+
+    it('pins the user-scope numbers the quotas page lists', () => {
+        expect(EVENTS_USER_HANDLERS_PER_SCOPE).toBe(20);
+        expect(EVENTS_USER_SCOPE_SOURCE_MAX_BYTES).toBe(1024 * 1024);
+        expect(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR).toBe(10);
+    });
+
+    it('keeps a user scope tighter than an app scope', () => {
+        expect(EVENTS_USER_HANDLERS_PER_SCOPE).toBeLessThan(
+            EVENTS_HANDLERS_PER_APP,
+        );
+        expect(EVENTS_USER_SCOPE_SOURCE_MAX_BYTES).toBeLessThan(
+            EVENTS_WORKER_SOURCE_MAX_BYTES,
+        );
+        expect(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR).toBeLessThan(
+            EVENTS_WORKER_DEPLOYS_PER_HOUR,
+        );
+    });
+
+    it('pins the rented-unit cap by plan', () => {
+        expect(RENTED_UNITS_PER_USER.limit).toBe(100);
+        expect(limitFor(RENTED_UNITS_PER_USER, DEFAULT_FREE_SUBSCRIPTION)).toBe(
+            16,
+        );
+        expect(limitFor(RENTED_UNITS_PER_USER, DEFAULT_TEMP_SUBSCRIPTION)).toBe(
+            16,
+        );
+    });
+});

src/backend/controllers/events/limits.ts

@@ -53,7 +53,12 @@ export interface TieredLimit {
     bySubscription: Record<string, number>;
 }
 
-const tiered = (paid: number, free: number, temp: number): TieredLimit => ({
+/** Paid, free and temporary caps as a {@link TieredLimit}. Called by extensions. */
+export const tiered = (
+    paid: number,
+    free: number,
+    temp: number,
+): TieredLimit => ({
     limit: paid,
     bySubscription: {
         [DEFAULT_FREE_SUBSCRIPTION]: free,
@@ -264,6 +269,12 @@ export const KV_SHARE_NOTIFY_LIMITS = [
  */
 export const EVENTS_HANDLERS_PER_APP = 100;
 
+/**
+ * Handlers one user may hold in their own scope of one app. Each is a script of
+ * its own, so the cap is tighter than the app's.
+ */
+export const EVENTS_USER_HANDLERS_PER_SCOPE = 20;
+
 /** Longest a serialized handler may be. */
 export const EVENTS_HANDLER_SOURCE_MAX_BYTES = 64 * 1024;
 
@@ -275,6 +286,9 @@ export const EVENTS_HANDLER_SOURCE_MAX_BYTES = 64 * 1024;
  */
 export const EVENTS_WORKER_SOURCE_MAX_BYTES = 5 * 1024 * 1024;
 
+/** Source one user scope may hold, all its handlers combined. */
+export const EVENTS_USER_SCOPE_SOURCE_MAX_BYTES = 1024 * 1024;
+
 /**
  * Deploys one app's events worker may trigger per hour.
  *
@@ -284,6 +298,16 @@ export const EVENTS_WORKER_SOURCE_MAX_BYTES = 5 * 1024 * 1024;
  */
 export const EVENTS_WORKER_DEPLOYS_PER_HOUR = 30;
 
+/** Deploys one user scope may trigger per hour. */
+export const EVENTS_USER_WORKER_DEPLOYS_PER_HOUR = 10;
+
+/**
+ * Units one user may rent at once: ordinary workers, user scopes, and the
+ * events workers of apps they own. Only creating a unit is refused; holdings
+ * over the cap are kept.
+ */
+export const RENTED_UNITS_PER_USER = tiered(100, 16, 16);
+
 /**
  * Handlers one `publishAll` may carry. A build step publishes its whole set in
  * one call, and the set is capped by what an app may hold anyway.
@@ -448,11 +472,13 @@ export const EVENTS_REGION_PENDING_CEILING = 1_000_000;
 // finish backs off the same way but is not the handler's failure, so it never
 // counts toward that run.
 
-/** Wait before the first retry of a delivery a handler could not take. */
-export const EVENTS_RETRY_BASE_MS = 2_000;
-
-/** Longest wait between retries, however many have failed. */
-export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
+// Wait before the first retry of a delivery a handler could not take, the
+// longest wait, and the doubling between them. Shared with system jobs.
+export {
+    deliveryBackoffMs,
+    EVENTS_RETRY_BASE_MS,
+    EVENTS_RETRY_MAX_MS,
+} from '../../util/retryBackoff.js';
 
 /**
  * Handler failures in a row before a subscription is suspended. Counted per
@@ -468,13 +494,6 @@ export const EVENTS_CONSECUTIVE_FAILURES = 5;
  */
 export const EVENTS_FAILURE_COUNTER_TTL_MS = 60 * 60 * 1000;
 
-/** How long to hold a delivery whose handler has failed `attempts` times. */
-export const deliveryBackoffMs = (attempts: number): number =>
-    Math.min(
-        EVENTS_RETRY_MAX_MS,
-        EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
-    );
-
 // -- Suspended backlog -----------------------------------------------
 //
 // A suspended subscription stops metering but keeps what it is owed, and that

src/backend/controllers/events/scheduleRoutes.test.ts

@@ -0,0 +1,274 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The schedule routes are thin delegates: without `services.schedules` every
+ * one answers 503 `events_disabled`; with it, each hands its request over and
+ * answers what it returns.
+ */
+
+import type { Request, Response } from 'express';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    ScheduleServiceContract,
+    ScheduleView,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { BroadcastController } from '../broadcast/BroadcastController.js';
+import type { EventsController } from './EventsController.js';
+
+let server: PuterServer;
+let events: EventsController;
+let broadcast: BroadcastController;
+let actor: Actor;
+let saved: unknown;
+
+const SCHEDULE_ID = '0b3a3c9e-9f61-4f0e-8d2a-2f1b6f8f2c11';
+
+const view = {
+    scheduleId: SCHEDULE_ID,
+    kind: 'recurring',
+    appUid: 'app-x',
+} as ScheduleView;
+
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+
+const fakeSchedules = () =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(async () => view),
+        list: vi.fn(async () => ({
+            items: [view],
+            cursor: 'next',
+            total: 3,
+        })),
+        get: vi.fn(async () => view),
+        remove: vi.fn(async () => {}),
+        receivePeerChanges: vi.fn(async () => ({ applied: 2 })),
+        prepareSendAt: vi.fn(),
+        createDeferredAppMessage: vi.fn(),
+    }) satisfies ScheduleServiceContract;
+
+const req = (init: {
+    body?: unknown;
+    query?: Record<string, unknown>;
+    params?: Record<string, string>;
+}): Request =>
+    ({
+        actor,
+        body: init.body ?? {},
+        query: init.query ?? {},
+        params: init.params ?? {},
+        headers: {},
+        rawBody: Buffer.from(JSON.stringify(init.body ?? {})),
+    }) as unknown as Request;
+
+const res = () => {
+    const sent: { status: number; body: unknown } = {
+        status: 200,
+        body: undefined,
+    };
+    const response = {
+        json: vi.fn((body: unknown) => {
+            sent.body = body;
+            return response;
+        }),
+        status: vi.fn((code: number) => {
+            sent.status = code;
+            return response;
+        }),
+        setHeader: vi.fn(() => response),
+    };
+    return { response: response as unknown as Response, sent };
+};
+
+/** Each route, called as the router would. */
+const routes: Array<[string, (response: Response) => Promise<void>]> = [
+    [
+        'POST /events/schedules',
+        (r) => events.createSchedule(req({ body: { every: 60_000 } }), r),
+    ],
+    [
+        'GET /events/schedules',
+        (r) => events.listSchedules(req({ query: { limit: '5' } }), r),
+    ],
+    [
+        'GET /events/schedules/:scheduleId',
+        (r) =>
+            events.getSchedule(req({ params: { scheduleId: SCHEDULE_ID } }), r),
+    ],
+    [
+        'POST /events/schedules/remove',
+        (r) =>
+            events.removeSchedule(
+                req({ body: { scheduleId: SCHEDULE_ID } }),
+                r,
+            ),
+    ],
+    [
+        'POST /broadcast/events/schedules',
+        (r) =>
+            broadcast.schedules(
+                req({ body: { namespace: 'n', changes: [] } }),
+                r,
+            ),
+    ],
+];
+
+beforeAll(async () => {
+    server = await setupTestServer();
+    events = server.controllers.events as unknown as EventsController;
+    broadcast = server.controllers.broadcast as unknown as BroadcastController;
+    actor = makeActor({
+        user: { id: 1, uuid: 'u-1', username: 'someone' },
+    } as never);
+    saved = slot().schedules;
+    vi.spyOn(
+        server.services.broadcast,
+        'verifySignedRequest',
+    ).mockResolvedValue({ ok: true, peerId: 'peer' } as never);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+});
+
+afterAll(async () => {
+    vi.restoreAllMocks();
+    await server?.shutdown();
+});
+
+describe('without services.schedules', () => {
+    it.each(routes)('%s answers 503 events_disabled', async (_name, call) => {
+        setSchedules(undefined);
+        const { response } = res();
+        let thrown: unknown;
+        try {
+            await call(response);
+        } catch (err) {
+            thrown = err;
+        }
+        expect(isHttpError(thrown)).toBe(true);
+        expect(thrown).toMatchObject({
+            statusCode: 503,
+            legacyCode: 'events_disabled',
+            // The designed answer, so it never pages.
+            noAlarm: true,
+        });
+    });
+
+    it('refuses an unsigned peer batch before saying schedules are off', async () => {
+        setSchedules(undefined);
+        vi.spyOn(
+            server.services.broadcast,
+            'verifySignedRequest',
+        ).mockResolvedValueOnce({
+            ok: false,
+            status: 401,
+            message: 'bad signature',
+        } as never);
+        const { response, sent } = res();
+        await broadcast.schedules(
+            req({ body: { namespace: 'n', changes: [] } }),
+            response,
+        );
+        expect(sent.status).toBe(401);
+    });
+});
+
+describe('with services.schedules', () => {
+    it('creates through createFor and answers its view', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.createSchedule(req({ body: { every: 60_000 } }), response);
+        expect(schedules.createFor).toHaveBeenCalledWith(actor, {
+            every: 60_000,
+        });
+        expect(sent.body).toEqual(view);
+    });
+
+    it('lists with the limit capped, the cursor and includeTotal', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.listSchedules(
+            req({
+                query: { limit: '5000', cursor: 'c', includeTotal: 'true' },
+            }),
+            response,
+        );
+        expect(schedules.list).toHaveBeenCalledWith(actor, {
+            limit: 200,
+            cursor: 'c',
+            includeTotal: true,
+        });
+        expect(sent.body).toEqual({ items: [view], cursor: 'next', total: 3 });
+    });
+
+    it('reads one through get', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.getSchedule(
+            req({ params: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.get).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual(view);
+    });
+
+    it('removes through remove and answers {}', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.removeSchedule(
+            req({ body: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.remove).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual({});
+    });
+
+    it('hands a signed peer batch to receivePeerChanges', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        const body = { namespace: 'n', changes: [] };
+        await broadcast.schedules(req({ body }), response);
+        expect(schedules.receivePeerChanges).toHaveBeenCalledWith(body);
+        expect(sent).toEqual({ status: 200, body: { ok: true, applied: 2 } });
+    });
+});

src/backend/controllers/events/workerDeploy.test.ts

@@ -25,9 +25,35 @@
 
 import MockRedis from 'ioredis-mock';
 import { describe, expect, it } from 'vitest';
-import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
-import { EventsWorkerDeployer } from './workerDeploy.js';
-import { eventsWorkerScript } from '../../services/events/workerRuntime.js';
+import type { fetch as undiciFetch } from 'undici';
+import {
+    DispatcherInvokeTransport,
+    EVENTS_DEPLOYED_HEADER,
+    EVENTS_DISPATCH_ERROR_HEADER,
+    EVENTS_HANDLED_HEADER,
+    EventsWorkerInvokerClient,
+    type EventsInvokeCall,
+} from '../../clients/events/EventsWorkerInvokerClient.js';
+import { INTERNAL_DEPLOY_TARGET } from '../../drivers/workers/WorkerDriver.js';
+import {
+    EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
+    EVENTS_WORKER_DEPLOYS_PER_HOUR,
+} from './limits.js';
+import {
+    EventsWorkerDeployer,
+    LocalEventsInvokeTransport,
+    type EventsDeployOptions,
+    type EventsDeployOutcome,
+} from './workerDeploy.js';
+import {
+    eventsWorkerScript,
+    userHandlerScript,
+} from '../../services/events/workerRuntime.js';
+import {
+    appHandlerScope,
+    hashContent,
+    type HandlerScope,
+} from '../../stores/events/EventHandlerStore.js';
 import { handlerSetHash } from '../../services/events/workerSource.js';
 import type { IConfig } from '../../types.js';
 
@@ -46,14 +72,22 @@ interface Owner {
 type Layers = ConstructorParameters<typeof EventsWorkerDeployer>[0];
 
 /** Stub layers whose only real dependency is a fresh redis mock. */
-const makeLayers = (owners: Record<string, Owner | null>) => {
+const makeLayers = (
+    owners: Record<string, Owner | null>,
+    events: Record<string, unknown> = {},
+) => {
     const redis = new MockRedis.Cluster(['redis://localhost:7001']);
     const created: string[] = [];
+    const targets: Array<Record<string, unknown>> = [];
     /** Apps whose handler sources were read, once per read. */
     const sourceReads: string[] = [];
     const layers = {
         config: {
-            events: { workerRuntime: true, internalSecret: 'sekret' },
+            events: {
+                workerRuntime: true,
+                internalSecret: 'sekret',
+                ...events,
+            },
             // `#namespace()` needs a local-worker shape to resolve without
             // throwing — this stub never reaches a real Cloudflare or
             // Miniflare deploy either way.
@@ -63,7 +97,7 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
         stores: {
             eventHandler: {
                 setForApp: async () => [{ name: 'h', sourceHash: SOURCE_HASH }],
-                allForApp: async (appUid: string) => {
+                allForApp: async ({ appUid }: { appUid: string }) => {
                     sourceReads.push(appUid);
                     return [
                         {
@@ -86,6 +120,18 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
                 },
             },
             user: {
+                getByUuid: async (uuid: string) => {
+                    const owner = Object.values(owners).find(
+                        (o) => `uuid-${o?.ownerUserId}` === uuid,
+                    );
+                    if (!owner) return null;
+                    return {
+                        id: owner.ownerUserId,
+                        uuid,
+                        username: `user-${owner.ownerUserId}`,
+                        suspended: owner.suspended ?? false,
+                    };
+                },
                 getById: async (id: number) => {
                     const owner = Object.values(owners).find(
                         (o) => o?.ownerUserId === id,
@@ -103,8 +149,12 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
         services: {},
         drivers: {
             workers: {
-                create: async (args: { workerName: string }) => {
+                create: async (args: {
+                    workerName: string;
+                    [INTERNAL_DEPLOY_TARGET]?: Record<string, unknown>;
+                }) => {
                     created.push(args.workerName);
+                    targets.push(args[INTERNAL_DEPLOY_TARGET] ?? {});
                     return { success: true, errors: [] };
                 },
             },
@@ -113,6 +163,7 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
     return {
         layers: layers as unknown as Layers,
         created,
+        targets,
         redis,
         sourceReads,
     };
@@ -131,8 +182,8 @@ describe('EventsWorkerDeployer.ensure', () => {
         const deployer = new EventsWorkerDeployer(layers);
 
         const [a, b] = await Promise.all([
-            deployer.ensure('app-a', SCRIPT),
-            deployer.ensure('app-b', SCRIPT),
+            deployer.ensure(appHandlerScope('app-a'), SCRIPT),
+            deployer.ensure(appHandlerScope('app-b'), SCRIPT),
         ]);
         expect(a).toBe('deployed');
         expect(b).toBe('no-owner');
@@ -149,7 +200,9 @@ describe('EventsWorkerDeployer.ensure', () => {
             String(EVENTS_WORKER_DEPLOYS_PER_HOUR),
         );
 
-        expect(await deployer.ensure('app-c', SCRIPT)).toBe('throttled');
+        expect(await deployer.ensure(appHandlerScope('app-c'), SCRIPT)).toBe(
+            'throttled',
+        );
         expect(sourceReads).toEqual([]);
         expect(created).toEqual([]);
     });
@@ -164,15 +217,680 @@ describe('EventsWorkerDeployer.ensure', () => {
             '',
         );
 
-        expect(await deployer.ensure('app-e', stale)).toBe('stale');
+        expect(await deployer.ensure(appHandlerScope('app-e'), stale)).toBe(
+            'stale',
+        );
         expect(sourceReads).toEqual([]);
         const hour = new Date().toISOString().slice(0, 13);
         expect(await redis.get(`ev:deploys:{app-e}:${hour}`)).toBeNull();
     });
 
+    it('passes events.workerCpuMs to the deploy target', async () => {
+        const { layers, targets } = makeLayers(
+            { 'app-f': { ownerUserId: 6 } },
+            { workerCpuMs: 50 },
+        );
+        const deployer = new EventsWorkerDeployer(layers);
+        expect(await deployer.ensure(appHandlerScope('app-f'), SCRIPT)).toBe(
+            'deployed',
+        );
+        expect(targets[0]!.cpuMs).toBe(50);
+    });
+
+    it('passes no cpuMs when events.workerCpuMs is unset', async () => {
+        const { layers, targets } = makeLayers({ 'app-g': { ownerUserId: 7 } });
+        const deployer = new EventsWorkerDeployer(layers);
+        expect(await deployer.ensure(appHandlerScope('app-g'), SCRIPT)).toBe(
+            'deployed',
+        );
+        expect(targets[0]!.cpuMs).toBeUndefined();
+    });
+
     it('deploys normally under the cap', async () => {
         const { layers } = makeLayers({ 'app-d': { ownerUserId: 4 } });
         const deployer = new EventsWorkerDeployer(layers);
-        expect(await deployer.ensure('app-d', SCRIPT)).toBe('deployed');
+        expect(await deployer.ensure(appHandlerScope('app-d'), SCRIPT)).toBe(
+            'deployed',
+        );
+    });
+});
+
+/** A server's schedules with a deploy gate that records and can refuse. */
+const withGate = (
+    layers: Layers,
+    refuse: (capped: boolean) => boolean = () => false,
+) => {
+    const gated: Array<{ script: string; capped: boolean }> = [];
+    const gate = {
+        gateDeploy: async (
+            script: string,
+            options: { capped: boolean },
+            deploy: () => Promise<EventsDeployOutcome>,
+        ): Promise<EventsDeployOutcome> => {
+            gated.push({ script, capped: options.capped });
+            return refuse(options.capped) ? 'busy' : deploy();
+        },
+    };
+    const deployer = new EventsWorkerDeployer({
+        ...layers,
+        services: { ...layers.services, schedules: gate },
+    } as unknown as Layers);
+    return { deployer, gated };
+};
+
+describe('the schedules` deploy gate', () => {
+    it('is skipped without schedules, or with schedules that have no gate', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        await expect(
+            new EventsWorkerDeployer(layers).ensure(
+                appHandlerScope('app-x'),
+                SCRIPT,
+                {
+                    lane: 'schedule',
+                },
+            ),
+        ).resolves.toBe('deployed');
+        const gateless = new EventsWorkerDeployer({
+            ...layers,
+            services: { ...layers.services, schedules: {} },
+        } as unknown as Layers);
+        await expect(
+            gateless.ensure(appHandlerScope('app-x'), SCRIPT, {
+                lane: 'schedule',
+            }),
+        ).resolves.toBe('deployed');
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('takes every deploy, capped only for a schedule run', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers);
+
+        await expect(
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT),
+        ).resolves.toBe('deployed');
+        await expect(
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
+                lane: 'schedule',
+            }),
+        ).resolves.toBe('deployed');
+        expect(gated).toEqual([
+            { script: SCRIPT, capped: false },
+            { script: SCRIPT, capped: true },
+        ]);
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('answers a refused schedule run busy, and a delivery still deploys', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer } = withGate(layers, (capped) => capped);
+
+        await expect(
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
+                lane: 'schedule',
+            }),
+        ).resolves.toBe('busy');
+        await expect(
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT),
+        ).resolves.toBe('deployed');
+        expect(created).toEqual([SCRIPT]);
+    });
+
+    it('lets deliveries that join a schedule run`s refused deploy deploy once', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const scheduled = deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
+            lane: 'schedule',
+        });
+        const deliveries = [
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT),
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT),
+        ];
+        await expect(scheduled).resolves.toBe('busy');
+        await expect(Promise.all(deliveries)).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(created).toEqual([SCRIPT]);
+        expect(gated.map((g) => g.capped)).toEqual([true, false]);
+    });
+
+    it('lets a schedule run join a delivery`s deploy in flight', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const delivery = deployer.ensure(appHandlerScope('app-x'), SCRIPT);
+        const scheduled = deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
+            lane: 'schedule',
+        });
+        await expect(Promise.all([delivery, scheduled])).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(gated).toEqual([{ script: SCRIPT, capped: false }]);
+        expect(created).toEqual([SCRIPT]);
+    });
+});
+
+describe('LocalEventsInvokeTransport', () => {
+    it('renders each request as it goes out, after a cold deploy', async () => {
+        const bodies: string[] = [];
+        let resident = false;
+        const transport = new LocalEventsInvokeTransport(
+            {
+                localworkerservice: {
+                    dispatchEventsWorker: async (request: Request) => {
+                        bodies.push(await request.text());
+                        if (!resident) return null;
+                        return new Response(null, {
+                            status: 200,
+                            headers: { 'x-puter-events-handled': '1' },
+                        });
+                    },
+                },
+            } as never,
+            {
+                ensure: async () => {
+                    resident = true;
+                    return 'deployed';
+                },
+                scopeFor: async (appUid: string) => appHandlerScope(appUid),
+            } as unknown as EventsWorkerDeployer,
+        );
+        let renders = 0;
+        const call: EventsInvokeCall = {
+            script: SCRIPT,
+            appUid: 'app-x',
+            key: 'k',
+            body: '{"n":0}',
+            timeoutMs: 5000,
+            render: () => `{"n":${++renders}}`,
+        };
+        await expect(transport.send(call)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(bodies).toEqual(['{"n":1}', '{"n":2}']);
+    });
+});
+
+describe('deploy-on-miss of an evicted script', () => {
+    it('redeploys the evicted script, then resends the run with the deployed marker', async () => {
+        const { layers, created } = makeLayers({ 'app-m': { ownerUserId: 8 } });
+        // The namespace only has the script once the deploy lands: before
+        // that the dispatcher answers a miss.
+        let resident = false;
+        layers.drivers.workers.create = (async (args: {
+            workerName: string;
+        }) => {
+            created.push(args.workerName);
+            resident = true;
+            return { success: true, errors: [] };
+        }) as never;
+        const deployer = new EventsWorkerDeployer(layers);
+
+        const sent: boolean[] = [];
+        const fetchImpl = (async (_url: string, init?: RequestInit) => {
+            const headers = (init?.headers ?? {}) as Record<string, string>;
+            sent.push(headers[EVENTS_DEPLOYED_HEADER] === '1');
+            if (resident)
+                return new Response(null, {
+                    status: 200,
+                    headers: { [EVENTS_HANDLED_HEADER]: '1' },
+                });
+            return new Response(null, {
+                status: 404,
+                headers: { [EVENTS_DISPATCH_ERROR_HEADER]: 'missing' },
+            });
+        }) as unknown as typeof undiciFetch;
+
+        const client = new EventsWorkerInvokerClient({
+            events: { workerRuntime: true },
+        } as IConfig);
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl,
+            }),
+        );
+        client.setMissHandler((appUid, script, options) =>
+            deployer.ensure(appHandlerScope(appUid), script, options),
+        );
+
+        const result = await client.invoke({
+            script: SCRIPT,
+            appUid: 'app-m',
+            handler: 'h',
+            token: 't',
+            key: 'k',
+            event: {},
+            ctx: {},
+        });
+
+        expect(result).toEqual({ outcome: 'settled', status: 200 });
+        expect(created).toEqual([SCRIPT]);
+        // One miss, then the resend carrying the deployed marker.
+        expect(sent).toEqual([false, true]);
+    });
+});
+
+// The redis mock is shared across tests, so each one uses its own app and code.
+let fresh = 0;
+const nextApp = () => `app-u${++fresh}`;
+const userScope = (holder: number, appUid: string) => ({ appUid, holder });
+const sourceN = (label: string) =>
+    `async () => { await ${label}${++fresh}(); }`;
+const scriptOf = (name: string, source: string) =>
+    userHandlerScript({ name, sourceHash: hashContent(source) }, '');
+
+type Sets = Record<number, Array<{ name: string; source: string }>>;
+
+/** Layers whose handler sets are per holder (0 is the app's own). */
+const withSets = (layers: Layers, sets: Sets) => {
+    const handlersOf = (scope: { appUid: string; holder: number }) =>
+        (sets[scope.holder] ?? []).map((h) => ({
+            appUid: scope.appUid,
+            name: h.name,
+            source: h.source,
+            sourceHash: hashContent(h.source),
+            createdAt: 0,
+            updatedAt: 0,
+        }));
+    const store = layers.stores.eventHandler as unknown as Record<
+        string,
+        unknown
+    >;
+    store.setForApp = async (scope: { appUid: string; holder: number }) =>
+        handlersOf(scope).map(({ name, sourceHash }) => ({ name, sourceHash }));
+    store.allForApp = async (scope: { appUid: string; holder: number }) =>
+        handlersOf(scope);
+};
+
+const holders = (...ids: number[]) =>
+    Object.fromEntries(
+        ids.map((id) => [`holder-${id}`, { ownerUserId: id }]),
+    ) as Record<string, Owner>;
+
+describe('EventsWorkerDeployer in a user scope', () => {
+    it('deploys the one handler whose script was asked for, as the holder', async () => {
+        const app = nextApp();
+        const a = sourceN('a');
+        const b = sourceN('b');
+        const { layers, created, targets } = makeLayers(holders(10));
+        withSets(layers, {
+            10: [
+                { name: 'a', source: a },
+                { name: 'b', source: b },
+            ],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+
+        const script = scriptOf('b', b);
+        expect(await deployer.ensure(userScope(10, app), script)).toBe(
+            'deployed',
+        );
+
+        expect(created).toEqual([script]);
+        const target = targets[0]!;
+        expect(target.tags).toEqual([`events-app:${app}`, 'user:uuid-10']);
+        // One handler's script: the other one's code is not in it.
+        expect(target.source).toContain(b);
+        expect(target.source).not.toContain(a);
+    });
+
+    it('answers a script no handler of the scope matches as stale, before the throttle', async () => {
+        const app = nextApp();
+        const { layers, redis, created } = makeLayers(holders(10));
+        withSets(layers, { 10: [{ name: 'a', source: sourceN('a') }] });
+        const deployer = new EventsWorkerDeployer(layers);
+
+        expect(
+            await deployer.ensure(
+                userScope(10, app),
+                scriptOf('b', sourceN('b')),
+            ),
+        ).toBe('stale');
+        expect(created).toEqual([]);
+        const hour = new Date().toISOString().slice(0, 13);
+        expect(await redis.get(`ev:deploys:{${app}}:10:${hour}`)).toBeNull();
+    });
+
+    it('answers a scope with no handlers as no-handlers', async () => {
+        const { layers } = makeLayers(holders(10));
+        withSets(layers, {});
+        const deployer = new EventsWorkerDeployer(layers);
+        expect(
+            await deployer.ensure(
+                userScope(10, nextApp()),
+                scriptOf('a', sourceN('a')),
+            ),
+        ).toBe('no-handlers');
+    });
+
+    it('does not deploy for a holder that is suspended or gone', async () => {
+        const source = sourceN('a');
+        const { layers, created } = makeLayers({
+            'holder-10': { ownerUserId: 10, suspended: true },
+        });
+        withSets(layers, {
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('a', source);
+        const app = nextApp();
+
+        expect(await deployer.ensure(userScope(10, app), script)).toBe(
+            'no-owner',
+        );
+        expect(await deployer.ensure(userScope(11, app), script)).toBe(
+            'no-owner',
+        );
+        expect(created).toEqual([]);
+    });
+
+    it('gives each scope its own hourly cap, apart from the app scope', async () => {
+        const app = nextApp();
+        const source = sourceN('a');
+        const { layers, created } = makeLayers({
+            ...holders(10, 11),
+            [app]: { ownerUserId: 12 },
+        });
+        withSets(layers, {
+            0: [{ name: 'a', source }],
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('a', source);
+
+        for (let i = 0; i < EVENTS_USER_WORKER_DEPLOYS_PER_HOUR; i++)
+            expect(await deployer.ensure(userScope(10, app), script)).toBe(
+                'deployed',
+            );
+        expect(await deployer.ensure(userScope(10, app), script)).toBe(
+            'throttled',
+        );
+        expect(created).toHaveLength(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR);
+
+        // Another holder of the same app, and the app's own scope, still go.
+        expect(await deployer.ensure(userScope(11, app), script)).toBe(
+            'deployed',
+        );
+        expect(await deployer.ensure(appHandlerScope(app), script)).toBe(
+            'deployed',
+        );
+    });
+
+    it('collapses concurrent deploys of one script across holders into one upload', async () => {
+        const app = nextApp();
+        const source = sourceN('a');
+        const { layers, created } = makeLayers(holders(10, 11));
+        withSets(layers, {
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('a', source);
+
+        await Promise.all([
+            deployer.ensure(userScope(10, app), script),
+            deployer.ensure(userScope(10, app), script),
+            deployer.ensure(userScope(11, app), script),
+            deployer.ensure(userScope(11, nextApp()), script),
+        ]);
+        expect(created).toHaveLength(1);
+    });
+
+    it('names a script for identical code the same under any holder and app', async () => {
+        const source = sourceN('a');
+        const { layers, created } = makeLayers(holders(10, 11));
+        withSets(layers, {
+            10: [{ name: '@x', source }],
+            11: [{ name: '@x', source }],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('@x', source);
+
+        expect(await deployer.ensure(userScope(10, nextApp()), script)).toBe(
+            'deployed',
+        );
+        expect(await deployer.ensure(userScope(11, nextApp()), script)).toBe(
+            'deployed',
+        );
+        expect(created).toEqual([script, script]);
+    });
+});
+
+describe('the deployed marker', () => {
+    const setup = () => {
+        const source = sourceN('a');
+        const made = makeLayers(holders(10, 11));
+        withSets(made.layers, {
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        return {
+            ...made,
+            deployer: new EventsWorkerDeployer(made.layers),
+            script: scriptOf('a', source),
+            app: nextApp(),
+        };
+    };
+
+    it('is left by a user-scope deploy with a ttl under a day', async () => {
+        const { deployer, redis, script, app } = setup();
+        await deployer.ensure(userScope(10, app), script);
+        const ttl = await redis.ttl(`ev:deployed:${script}`);
+        expect(ttl).toBeGreaterThan(0);
+        expect(ttl).toBeLessThan(24 * 60 * 60);
+    });
+
+    it('lets an eager deploy skip the upload another holder already made', async () => {
+        const { deployer, created, script, app } = setup();
+        expect(await deployer.deployEagerly(userScope(10, app), script)).toBe(
+            'deployed',
+        );
+        expect(await deployer.deployEagerly(userScope(11, app), script)).toBe(
+            'deployed',
+        );
+        expect(created).toEqual([script]);
+    });
+
+    it('spends no throttle on a skipped eager deploy', async () => {
+        const { deployer, redis, script, app } = setup();
+        await deployer.deployEagerly(userScope(10, app), script);
+        await deployer.deployEagerly(userScope(11, app), script);
+        const hour = new Date().toISOString().slice(0, 13);
+        expect(await redis.get(`ev:deploys:{${app}}:11:${hour}`)).toBeNull();
+    });
+
+    it('never skips a deploy-on-miss', async () => {
+        const { deployer, created, script, app } = setup();
+        await deployer.deployEagerly(userScope(10, app), script);
+        expect(await deployer.ensure(userScope(11, app), script)).toBe(
+            'deployed',
+        );
+        expect(created).toEqual([script, script]);
+    });
+
+    it('is not left by an app-scope deploy', async () => {
+        const app = nextApp();
+        const source = sourceN('a');
+        const { layers, redis, created } = makeLayers({
+            [app]: { ownerUserId: 4 },
+        });
+        withSets(layers, { 0: [{ name: 'a', source }] });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('a', source);
+        await deployer.ensure(appHandlerScope(app), script);
+        expect(created).toEqual([script]);
+        expect(await redis.get(`ev:deployed:${script}`)).toBeNull();
+    });
+
+    it('still deploys eagerly when the marker cannot be read', async () => {
+        const { deployer, redis, created, script, app } = setup();
+        await deployer.deployEagerly(userScope(10, app), script);
+        const get = redis.get.bind(redis);
+        redis.get = (async (key: string) => {
+            if (key.startsWith('ev:deployed:')) throw new Error('down');
+            return get(key);
+        }) as never;
+        expect(await deployer.deployEagerly(userScope(11, app), script)).toBe(
+            'deployed',
+        );
+        expect(created).toEqual([script, script]);
+    });
+});
+
+describe('scopeFor', () => {
+    it('reads no holder as the app scope, and resolves a uuid to the user scope', async () => {
+        const { layers } = makeLayers(holders(10));
+        const deployer = new EventsWorkerDeployer(layers);
+        expect(await deployer.scopeFor('app-u')).toEqual({
+            appUid: 'app-u',
+            holder: 0,
+        });
+        expect(await deployer.scopeFor('app-u', '')).toEqual({
+            appUid: 'app-u',
+            holder: 0,
+        });
+        expect(await deployer.scopeFor('app-u', 'uuid-10')).toEqual({
+            appUid: 'app-u',
+            holder: 10,
+        });
+        expect(await deployer.scopeFor('app-u', 'uuid-99')).toBeNull();
+    });
+});
+
+describe('LocalEventsInvokeTransport in a user scope', () => {
+    it('deploys the holder`s script on a miss and refuses an unknown holder', async () => {
+        const source = sourceN('a');
+        const { layers, created } = makeLayers(holders(10));
+        withSets(layers, { 10: [{ name: 'a', source }] });
+        const deployer = new EventsWorkerDeployer(layers);
+        let resident = false;
+        const transport = new LocalEventsInvokeTransport(
+            {
+                localworkerservice: {
+                    dispatchEventsWorker: async () => {
+                        if (!resident) return null;
+                        return new Response(null, {
+                            status: 200,
+                            headers: { 'x-puter-events-handled': '1' },
+                        });
+                    },
+                },
+            } as never,
+            {
+                ensure: async (
+                    scope: HandlerScope,
+                    script: string,
+                    options: EventsDeployOptions,
+                ) => {
+                    const outcome = await deployer.ensure(
+                        scope,
+                        script,
+                        options,
+                    );
+                    resident = outcome === 'deployed';
+                    return outcome;
+                },
+                scopeFor: (appUid: string, holder?: string) =>
+                    deployer.scopeFor(appUid, holder),
+            } as unknown as EventsWorkerDeployer,
+        );
+        const script = scriptOf('a', source);
+        const call: EventsInvokeCall = {
+            script,
+            appUid: nextApp(),
+            holder: 'uuid-10',
+            key: 'k',
+            body: '{}',
+            timeoutMs: 5000,
+        };
+
+        await expect(transport.send(call)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(created).toEqual([script]);
+
+        resident = false;
+        await expect(
+            transport.send({ ...call, holder: 'uuid-99' }),
+        ).resolves.toMatchObject({ status: null, error: 'deploy: no-holder' });
+        expect(created).toEqual([script]);
+    });
+});
+
+describe('concurrent eager deploys of one script', () => {
+    const setup = () => {
+        const source = sourceN('a');
+        const made = makeLayers(holders(10, 11));
+        withSets(made.layers, {
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        return {
+            ...made,
+            script: scriptOf('a', source),
+            app: nextApp(),
+        };
+    };
+
+    it('upload once when two holders publish the same new code together', async () => {
+        const { layers, created, script, app } = setup();
+        const deployer = new EventsWorkerDeployer(layers);
+
+        const outcomes = await Promise.all([
+            deployer.deployEagerly(userScope(10, app), script),
+            deployer.deployEagerly(userScope(11, app), script),
+        ]);
+
+        expect(outcomes).toEqual(['deployed', 'deployed']);
+        expect(created).toEqual([script]);
+    });
+
+    it('skip the upload when the marker appears while they wait on the gate', async () => {
+        const { layers, redis, created, script, app } = setup();
+        // Another node's deploy of the script finishes before the gate opens.
+        const gated = new EventsWorkerDeployer({
+            ...layers,
+            services: {
+                schedules: {
+                    gateDeploy: async (
+                        _script: string,
+                        _options: { capped: boolean },
+                        deploy: () => Promise<EventsDeployOutcome>,
+                    ) => {
+                        await redis.set(`ev:deployed:${script}`, '1');
+                        return deploy();
+                    },
+                },
+            },
+        } as unknown as Layers);
+
+        expect(
+            await gated.ensure(userScope(10, app), script, { eager: true }),
+        ).toBe('deployed');
+        expect(created).toEqual([]);
+        // A miss still uploads, whatever the marker says.
+        expect(await gated.ensure(userScope(11, app), script)).toBe('deployed');
+        expect(created).toEqual([script]);
+    });
+
+    it('make a miss that joins an eager deploy deploy after it', async () => {
+        const { layers, redis, created, script, app } = setup();
+        const deployer = new EventsWorkerDeployer(layers);
+        await redis.set(`ev:deployed:${script}`, '1');
+
+        const eager = deployer.ensure(userScope(10, app), script, {
+            eager: true,
+        });
+        const miss = deployer.ensure(userScope(11, app), script);
+
+        expect(await eager).toBe('deployed');
+        expect(await miss).toBe('deployed');
+        // The eager one skipped on the marker; the miss uploaded.
+        expect(created).toEqual([script]);
     });
 });

src/backend/controllers/events/workerDeploy.ts

@@ -29,6 +29,7 @@ import {
 import type {
     EventsInvokeAnswer,
     EventsInvokeCall,
+    InvokeLane,
 } from '../../clients/events/EventsWorkerInvokerClient.js';
 import {
     EVENTS_INVOKE_PATH,
@@ -36,47 +37,89 @@ import {
     failedAnswer,
 } from '../../clients/events/EventsWorkerInvokerClient.js';
 import type { puterServices } from '../../services/index.js';
+import type { IExtensionServiceInstances } from '../../services/types.js';
 import {
     eventsInvokeKey,
     eventsWorkerScope,
     eventsWorkerScript,
+    userHandlerScript,
 } from '../../services/events/workerRuntime.js';
 import {
     generateEventsWorkerSource,
     handlerSetHash,
 } from '../../services/events/workerSource.js';
+import {
+    appHandlerScope,
+    type HandlerScope,
+} from '../../stores/events/EventHandlerStore.js';
 import type { puterStores } from '../../stores/index.js';
 import type { IConfig, LayerInstances } from '../../types.js';
-import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
+import {
+    EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
+    EVENTS_WORKER_DEPLOYS_PER_HOUR,
+} from './limits.js';
 
 /**
  * Deploys an app's events worker from its published handlers.
  *
  * Deployed the first time a delivery needs it and again on eviction — a publish
- * only writes rows. Scripts carry no worker token and no `subdomains` row: not
- * one of the owner's workers, not addressable from the internet, not listed or
- * deleted as if it were. A republished set leaves the old script behind rather
- * than deleting it; redeploying a still-live one is always safe.
+ * only writes rows (a user-scope publish also deploys eagerly). An app's scope
+ * is one script for the whole set; a user scope is one script per handler,
+ * deployed as the holder. Scripts carry no worker token and no `subdomains`
+ * row: not one of the owner's workers, not addressable from the internet, not
+ * listed or deleted as if it were. A republished set leaves the old script
+ * behind rather than deleting it; redeploying a still-live one is always safe.
  */
 
 /** Why a deploy could not happen, for the callers that answer differently. */
 export type EventsDeployOutcome =
-    'deployed' | 'stale' | 'no-handlers' | 'no-owner' | 'throttled' | 'failed';
+    | 'deployed'
+    | 'stale'
+    | 'no-handlers'
+    | 'no-owner'
+    | 'throttled'
+    | 'failed'
+    | 'busy';
+
+/** Whose call asked for the deploy: a schedule run's is held to a share. */
+export interface EventsDeployOptions {
+    lane?: InvokeLane;
+    /**
+     * A user-scope publish's deploy, which skips the upload when the script's
+     * deployed marker is already set. A deploy-on-miss is never eager.
+     */
+    eager?: boolean;
+}
 
 interface DeployLayers {
     config: IConfig;
     clients: LayerInstances<typeof puterClients>;
     stores: LayerInstances<typeof puterStores>;
-    services: LayerInstances<typeof puterServices>;
+    services: LayerInstances<typeof puterServices> &
+        Pick<IExtensionServiceInstances, 'schedules'>;
     drivers: LayerInstances<typeof puterDrivers>;
 }
 
 /**
- * `ev:deploys:{<appUid>}:<hour>` bucket TTL — comfortably past the hour it
- * counts.
+ * `ev:deploys:{<appUid>}[:<holder>]:<hour>` bucket TTL — comfortably past the
+ * hour it counts.
  */
 const DEPLOY_THROTTLE_TTL_SECONDS = 2 * 60 * 60;
 
+/** How long `ev:deployed:<script>` says a script was uploaded. Under a day. */
+const DEPLOYED_MARKER_TTL_SECONDS = 20 * 60 * 60;
+
+const deployedMarkerKey = (script: string): string => `ev:deployed:${script}`;
+
+const isUserScope = (scope: HandlerScope): boolean => scope.holder !== 0;
+
+interface InFlightDeploy {
+    /** A schedule run's, which the region's deploy slots may refuse. */
+    capped: boolean;
+    eager: boolean;
+    promise: Promise<EventsDeployOutcome>;
+}
+
 interface DeployResult {
     success?: boolean;
     errors?: unknown[];
@@ -85,7 +128,7 @@ interface DeployResult {
 export class EventsWorkerDeployer {
     readonly #layers: DeployLayers;
     /** Collapses the burst of misses one cold app produces into one deploy. */
-    readonly #inFlight = new Map<string, Promise<EventsDeployOutcome>>();
+    readonly #inFlight = new Map<string, InFlightDeploy>();
 
     constructor(layers: DeployLayers) {
         this.#layers = layers;
@@ -100,44 +143,188 @@ export class EventsWorkerDeployer {
     }
 
     /**
-     * Deploy `script`, which must be what the app's handlers currently hash to.
-     * A request for anything else is `stale` — the set moved on, and the next
-     * delivery resolves the new script rather than this one being revived.
+     * Deploy `script`, which must be what the scope's handlers currently hash
+     * to: the whole set for an app, one handler for a user scope. A request for
+     * anything else is `stale` — the handlers moved on, and the next delivery
+     * resolves the new script rather than this one being revived. Always
+     * uploads; a deploy-on-miss means whatever said it was there is wrong.
      */
-    ensure(appUid: string, script: string): Promise<EventsDeployOutcome> {
-        const key = `${appUid}|${script}`;
+    ensure(
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions = {},
+    ): Promise<EventsDeployOutcome> {
+        // A user scope's script is the same bytes whoever holds it, so
+        // concurrent deploys of it from any holder or app are one.
+        const key = isUserScope(scope)
+            ? `user|${script}`
+            : `${scope.appUid}|${scope.holder}|${script}`;
+        const capped = options.lane === 'schedule';
         const existing = this.#inFlight.get(key);
-        if (existing) return existing;
-        const started = this.#deploy(appUid, script).finally(() =>
-            this.#inFlight.delete(key),
+        if (!existing) return this.#start(key, scope, script, options);
+        // An eager deploy may have skipped the upload, which a miss must not
+        // take for done.
+        if (existing.eager && !options.eager)
+            return this.#afterEager(
+                existing.promise,
+                key,
+                scope,
+                script,
+                options,
+            );
+        if (capped || !existing.capped) return existing.promise;
+        return this.#afterCapped(existing.promise, key, scope, script, options);
+    }
+
+    /**
+     * {@link ensure} for the deploy a user-scope publish starts, which skips the
+     * upload when another holder's deploy of the same script left its marker.
+     * Only this path reads the marker.
+     */
+    async deployEagerly(
+        scope: HandlerScope,
+        script: string,
+    ): Promise<EventsDeployOutcome> {
+        if (await this.#markedDeployed(script)) return 'deployed';
+        return this.ensure(scope, script, { eager: true });
+    }
+
+    /**
+     * The scope a script request names: the app's own, or the user scope of
+     * `holder` (a user uuid). Null for a holder that does not exist.
+     */
+    async scopeFor(
+        appUid: string,
+        holder?: string | null,
+    ): Promise<HandlerScope | null> {
+        if (!holder) return appHandlerScope(appUid);
+        const user = await this.#layers.stores.user.getByUuid(holder);
+        if (!user?.id) return null;
+        return { appUid, holder: Number(user.id) };
+    }
+
+    #start(
+        key: string,
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const entry: InFlightDeploy = {
+            capped: options.lane === 'schedule',
+            eager: options.eager === true,
+            promise: this.#gated(scope, script, options).finally(() => {
+                if (this.#inFlight.get(key) === entry)
+                    this.#inFlight.delete(key);
+            }),
+        };
+        this.#inFlight.set(key, entry);
+        return entry.promise;
+    }
+
+    /** A miss that found an eager deploy in flight deploys after it. */
+    async #afterEager(
+        eager: Promise<EventsDeployOutcome>,
+        key: string,
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        await eager;
+        const current = this.#inFlight.get(key);
+        if (current && !current.eager) return current.promise;
+        return this.#start(key, scope, script, options);
+    }
+
+    /**
+     * A delivery that joined a schedule run's deploy: `busy` means no deploy
+     * started, and a delivery is never refused a slot, so it asks again.
+     */
+    async #afterCapped(
+        capped: Promise<EventsDeployOutcome>,
+        key: string,
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const outcome = await capped;
+        if (outcome !== 'busy') return outcome;
+        const current = this.#inFlight.get(key);
+        if (current && !current.capped) return current.promise;
+        return this.#start(key, scope, script, options);
+    }
+
+    /**
+     * Every deploy goes through the schedules' deploy gate when a server has
+     * one: a per-script lock, so a burst of runs of an evicted script deploys
+     * it once, and a region-wide cap that a schedule run's deploy (`capped`)
+     * may be refused with `busy`. A delivery's deploy is never refused.
+     */
+    #gated(
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const deploy = () =>
+            this.#deploy(scope, script, options.eager === true);
+        return (
+            this.#layers.services.schedules?.gateDeploy?.(
+                script,
+                { capped: options.lane === 'schedule' },
+                deploy,
+            ) ?? deploy()
         );
-        this.#inFlight.set(key, started);
-        return started;
     }
 
     async #deploy(
-        appUid: string,
+        handlerScope: HandlerScope,
         script: string,
+        eager: boolean,
     ): Promise<EventsDeployOutcome> {
+        const { appUid } = handlerScope;
         const { stores, drivers } = this.#layers;
-        const scope = eventsWorkerScope(this.#layers.config);
+        const backend = eventsWorkerScope(this.#layers.config);
+        const userScope = isUserScope(handlerScope);
 
         // Names and hashes are enough to tell a stale request, so the set's
         // sources are read and built only once the throttle has let it through.
-        const set = await stores.eventHandler.setForApp(appUid);
+        const set = await stores.eventHandler.setForApp(handlerScope);
         if (set.length === 0) return 'no-handlers';
         // Two apps that published byte-identical sets share one script, which
         // is sound: the source is the same, and everything per-app arrives with
-        // the invocation. Whoever deploys first owns it.
-        if (eventsWorkerScript(handlerSetHash(set), scope) !== script)
+        // the invocation. Whoever deploys first owns it. A user scope's script
+        // is one handler's, so it is shared the same way across holders.
+        const wanted = userScope
+            ? set.find((entry) => userHandlerScript(entry, backend) === script)
+            : undefined;
+        if (
+            userScope
+                ? !wanted
+                : eventsWorkerScript(handlerSetHash(set), backend) !== script
+        )
             return 'stale';
 
-        if (await this.#tooManyDeploysThisHour(appUid)) return 'throttled';
+        // Asked again inside the gate, where another deploy of this script may
+        // have just finished.
+        if (eager && userScope && (await this.#markedDeployed(script)))
+            return 'deployed';
 
-        const handlers = await stores.eventHandler.allForApp(appUid);
+        if (await this.#tooManyDeploysThisHour(handlerScope))
+            return 'throttled';
+
+        const all = await stores.eventHandler.allForApp(handlerScope);
+        const handlers = wanted
+            ? all.filter(
+                  (handler) =>
+                      handler.name === wanted.name &&
+                      handler.sourceHash === wanted.sourceHash,
+              )
+            : all;
         const generated = generateEventsWorkerSource(handlers);
         // The set can move between the two reads.
-        if (eventsWorkerScript(generated.setHash, scope) !== script)
+        if (
+            handlers.length === 0 ||
+            eventsWorkerScript(generated.setHash, backend) !== script
+        )
             return 'stale';
         if (generated.tooLarge) {
             console.error(
@@ -151,7 +338,7 @@ export class EventsWorkerDeployer {
                 generated.broken.join(', '),
             );
 
-        const owner = await this.#owner(appUid);
+        const owner = await this.#owner(handlerScope);
         if (!owner) return 'no-owner';
 
         const secret = this.#layers.config.events?.internalSecret;
@@ -180,6 +367,7 @@ export class EventsWorkerDeployer {
                         runtime: 'events',
                         source: generated.source,
                         omitOwnerToken: true,
+                        cpuMs: this.#layers.config.events?.workerCpuMs,
                         // What the script compares an invocation against.
                         // Derived from the script name, so a redeploy of
                         // the same set binds the same key.
@@ -210,34 +398,74 @@ export class EventsWorkerDeployer {
             );
             return 'failed';
         }
+        if (userScope) await this.#markDeployed(script);
         return 'deployed';
     }
 
-    /**
-     * Whether this app has already deployed `EVENTS_WORKER_DEPLOYS_PER_HOUR`
-     * times this hour. A flapping set (publish, remove, republish) provisions
-     * upstream on every miss, and this is the ceiling on that loop.
-     */
-    async #tooManyDeploysThisHour(appUid: string): Promise<boolean> {
-        const redis = this.#layers.clients.redis;
-        const hour = new Date().toISOString().slice(0, 13);
-        const key = `ev:deploys:{${appUid}}:${hour}`;
-        const count = await redis.incr(key);
-        if (count === 1) await redis.expire(key, DEPLOY_THROTTLE_TTL_SECONDS);
-        return count > EVENTS_WORKER_DEPLOYS_PER_HOUR;
+    async #markedDeployed(script: string): Promise<boolean> {
+        try {
+            return !!(await this.#layers.clients.redis.get(
+                deployedMarkerKey(script),
+            ));
+        } catch (err) {
+            console.warn('[events] deployed marker unreadable', err);
+            return false;
+        }
     }
 
-    /** The app owner, as an actor the deploy can run as. */
+    async #markDeployed(script: string): Promise<void> {
+        try {
+            await this.#layers.clients.redis.set(
+                deployedMarkerKey(script),
+                '1',
+                'EX',
+                DEPLOYED_MARKER_TTL_SECONDS,
+            );
+        } catch (err) {
+            console.warn('[events] could not mark a script deployed', err);
+        }
+    }
+
+    /**
+     * Whether this scope has already deployed its hourly cap this hour. A
+     * flapping set (publish, remove, republish) provisions upstream on every
+     * miss, and this is the ceiling on that loop. An app's counter is per app;
+     * a user scope has its own, so one user cannot spend another's.
+     */
+    async #tooManyDeploysThisHour(scope: HandlerScope): Promise<boolean> {
+        const redis = this.#layers.clients.redis;
+        const hour = new Date().toISOString().slice(0, 13);
+        const userScope = isUserScope(scope);
+        const key = userScope
+            ? `ev:deploys:{${scope.appUid}}:${scope.holder}:${hour}`
+            : `ev:deploys:{${scope.appUid}}:${hour}`;
+        const count = await redis.incr(key);
+        if (count === 1) await redis.expire(key, DEPLOY_THROTTLE_TTL_SECONDS);
+        return (
+            count >
+            (userScope
+                ? EVENTS_USER_WORKER_DEPLOYS_PER_HOUR
+                : EVENTS_WORKER_DEPLOYS_PER_HOUR)
+        );
+    }
+
+    /**
+     * The user a scope's deploy runs as, as an actor: the app's owner, or the
+     * holder of a user scope.
+     */
     async #owner(
-        appUid: string,
+        scope: HandlerScope,
     ): Promise<
         | (Actor & { user: { id: number; uuid: string; username: string } })
         | null
     > {
         const { stores } = this.#layers;
-        const app = await stores.app.getByUid(appUid);
-        const ownerUserId = Number(app?.owner_user_id);
-        if (!app || !Number.isFinite(ownerUserId)) return null;
+        let ownerUserId = scope.holder;
+        if (!isUserScope(scope)) {
+            const app = await stores.app.getByUid(scope.appUid);
+            ownerUserId = Number(app?.owner_user_id);
+            if (!app || !Number.isFinite(ownerUserId)) return null;
+        }
 
         const user = await stores.user.getById(ownerUserId);
         if (!user?.id || !user.uuid || !user.username || user.suspended)
@@ -299,6 +527,8 @@ export class LocalEventsInvokeTransport {
     }
 
     async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        // Built per attempt, so a send time stamped into the body is taken
+        // after a cold deploy rather than before it.
         const request = (): Request =>
             new Request(
                 `http://${call.script}.workers.puter.localhost${EVENTS_INVOKE_PATH}`,
@@ -308,7 +538,7 @@ export class LocalEventsInvokeTransport {
                         'content-type': 'application/json',
                         'x-puter-events-key': call.key,
                     },
-                    body: call.body,
+                    body: call.render ? call.render() : call.body,
                 },
             );
 
@@ -316,12 +546,20 @@ export class LocalEventsInvokeTransport {
             const resident = await this.#dispatch(request(), call.timeoutMs);
             if (resident !== null) return resident;
 
-            const outcome = await this.#deployer.ensure(
+            const scope = await this.#deployer.scopeFor(
                 call.appUid,
-                call.script,
+                call.holder,
             );
+            if (!scope) return { status: null, error: 'deploy: no-holder' };
+            const outcome = await this.#deployer.ensure(scope, call.script, {
+                lane: call.lane,
+            });
             if (outcome !== 'deployed')
-                return { status: null, error: `deploy: ${outcome}` };
+                return {
+                    status: null,
+                    error: `deploy: ${outcome}`,
+                    ...(outcome === 'busy' ? { deployBusy: true } : {}),
+                };
 
             const retried = await this.#dispatch(request(), call.timeoutMs);
             if (retried === null)

src/backend/controllers/homepage/HomepageController.test.ts

@@ -78,11 +78,13 @@ interface CapturedResponse {
     statusCode: number;
     body: unknown;
     contentType?: string;
+    location?: string;
 }
 
 const makeReq = (init: {
     params?: Record<string, unknown>;
     path?: string;
+    originalUrl?: string;
     actor?: Actor;
     hostname?: string;
     protocol?: string;
@@ -93,6 +95,7 @@ const makeReq = (init: {
         headers: {},
         params: init.params ?? {},
         path: init.path ?? '/',
+        originalUrl: init.originalUrl ?? init.path ?? '/',
         hostname: init.hostname ?? 'test.local',
         protocol: init.protocol ?? 'http',
         actor: init.actor,
@@ -130,6 +133,11 @@ const makeRes = () => {
             captured.contentType = value;
             return res;
         }),
+        redirect: vi.fn((code: number, location: string) => {
+            captured.statusCode = code;
+            captured.location = location;
+            return res;
+        }),
     };
     return { res: res as unknown as Response, captured };
 };
@@ -213,14 +221,12 @@ describe('HomepageController shell routes', () => {
         expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
     });
 
-    it('serves the shell on /settings, /dashboard, /desktop, /action, /@:username', async () => {
+    it('serves the shell on /settings, /dashboard, /action, /@:username', async () => {
         for (const path of [
             '/settings',
             '/settings/*splat',
             '/dashboard',
             '/dashboard/',
-            '/desktop',
-            '/desktop/',
             '/action/*splat',
             '/@:username',
         ]) {
@@ -230,6 +236,18 @@ describe('HomepageController shell routes', () => {
             expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
         }
     });
+
+    it('serves the shell on a path inside a public folder, /@:username/*splat', async () => {
+        const { res, captured } = makeRes();
+        await callRoute(
+            'get',
+            '/@:username/*splat',
+            makeReq({ path: '/@alice/photos/cat.png' }),
+            res,
+        );
+        expect(captured.statusCode).toBe(200);
+        expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
+    });
 });
 
 // ── /app/:name ──────────────────────────────────────────────────────
@@ -359,49 +377,6 @@ describe('HomepageController GET /app/:name', () => {
         expect(html).not.toContain('owner_user_id');
     });
 
-    it('serves the same app shell under /desktop/app/:name', async () => {
-        const { userId } = await makeUser();
-        const name = `desk-${Math.random().toString(36).slice(2, 10)}`;
-        await server.stores.app.create(
-            {
-                name,
-                title: 'Desktop App',
-                description: 'opens on the desktop instead of the dashboard',
-                index_url: `https://example.com/${name}/`,
-                approved_for_listing: 1,
-            },
-            { ownerUserId: userId },
-        );
-
-        const { res, captured } = makeRes();
-        await callRoute(
-            'get',
-            '/desktop/app/:name',
-            makeReq({ params: { name }, path: `/desktop/app/${name}` }),
-            res,
-        );
-
-        expect(captured.statusCode).toBe(200);
-        const html = String(captured.body);
-        expect(html).toMatch(/<!DOCTYPE html>/i);
-        expect(html).toContain('Desktop App');
-    });
-
-    it('returns 404 under /desktop/app/:name when the app is unknown', async () => {
-        const { res, captured } = makeRes();
-        await callRoute(
-            'get',
-            '/desktop/app/:name',
-            makeReq({
-                params: { name: 'no-such-app' },
-                path: '/desktop/app/no-such-app',
-            }),
-            res,
-        );
-        expect(captured.statusCode).toBe(404);
-        expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
-    });
-
     it('omits index_url even for a public app — the shell is not the launch authority', async () => {
         const { userId } = await makeUser();
         const name = `pub-${Math.random().toString(36).slice(2, 10)}`;
@@ -433,8 +408,8 @@ describe('HomepageController GET /app/:name', () => {
 // -- app landing rate limit --
 
 describe('HomepageController app landing rate limit', () => {
-    it('rate-limits /app/:name and /desktop/app/:name per network', () => {
-        for (const path of ['/app/:name', '/desktop/app/:name']) {
+    it('rate-limits /app/:name per network', () => {
+        for (const path of ['/app/:name']) {
             const route = router.routes.find(
                 (r) => r.method === 'get' && r.path === path,
             );
@@ -452,7 +427,7 @@ describe('HomepageController app landing rate limit', () => {
 // ── /show/* ─────────────────────────────────────────────────────────
 
 describe('HomepageController GET /show/*splat', () => {
-    it('emits a launch_app explorer call with the post-/show path', async () => {
+    it('serves the plain shell; the GUI opens the path itself', async () => {
         const { res, captured } = makeRes();
         await callRoute(
             'get',
@@ -460,13 +435,52 @@ describe('HomepageController GET /show/*splat', () => {
             makeReq({ path: '/show/alice/Documents' }),
             res,
         );
-        // The launch payload is JSON-serialized into the rendered HTML.
-        // We assert the rendered shell included the explorer launch hint
-        // pointing at the expected (slashed) path.
         const html = String(captured.body);
-        expect(html).toContain('launch_app');
-        expect(html).toContain('explorer');
-        expect(html).toContain('/alice/Documents');
+        expect(html).toMatch(/<!DOCTYPE html>/i);
+        // No boot-time launch hook: the one it used to carry named a GUI
+        // global that no longer exists, which stopped the GUI booting.
+        expect(html).not.toContain('launch_app');
+    });
+});
+
+// ── /desktop ────────────────────────────────────────────────────────
+
+describe('HomepageController GET /desktop', () => {
+    it('redirects /desktop to the root, keeping the query', async () => {
+        for (const path of ['/desktop', '/desktop/']) {
+            const { res, captured } = makeRes();
+            await callRoute('get', path, makeReq({ path }), res);
+            expect(captured.statusCode).toBe(302);
+            expect(captured.location).toBe('/');
+        }
+
+        const { res, captured } = makeRes();
+        await callRoute(
+            'get',
+            '/desktop',
+            makeReq({
+                path: '/desktop',
+                originalUrl: '/desktop?shared=%2Falice%2Fx%2Fy&download=a',
+            }),
+            res,
+        );
+        expect(captured.location).toBe('/?shared=%2Falice%2Fx%2Fy&download=a');
+    });
+
+    it('redirects /desktop/app/:name to the app landing', async () => {
+        const { res, captured } = makeRes();
+        await callRoute(
+            'get',
+            '/desktop/app/:name',
+            makeReq({
+                params: { name: 'editor' },
+                path: '/desktop/app/editor',
+                originalUrl: '/desktop/app/editor?file=%2Falice%2Fa.txt',
+            }),
+            res,
+        );
+        expect(captured.statusCode).toBe(302);
+        expect(captured.location).toBe('/app/editor?file=%2Falice%2Fa.txt');
     });
 });
 

src/backend/controllers/homepage/HomepageController.ts

@@ -25,7 +25,6 @@ import type { PuterRouter } from '../../core/http/PuterRouter';
 import type {
     PuterHomepageService,
     PageMeta,
-    LaunchOptions,
 } from '../../services/homepage/PuterHomepageService';
 
 /** Longest description/og:description/twitter:description on `/app/<name>`. */
@@ -43,9 +42,9 @@ const APP_LANDING_LIMIT = {
 };
 
 /**
- * App descriptions can run long; search snippets and social cards want a
- * short blurb. Collapses whitespace, then cuts at a word boundary with an
- * ellipsis so the result never exceeds `APP_META_DESCRIPTION_MAX`.
+ * App descriptions can run long; search snippets and social cards want a short
+ * blurb. Collapses whitespace, then cuts at a word boundary with an ellipsis so
+ * the result never exceeds `APP_META_DESCRIPTION_MAX`.
  */
 export function appMetaDescription(text: string): string {
     const clean = text.replace(/\s+/g, ' ').trim();
@@ -89,7 +88,6 @@ export class HomepageController extends PuterController {
             req: express.Request,
             res: express.Response,
             metaOverrides: Partial<PageMeta> = {},
-            launch: LaunchOptions = {},
         ) => {
             const meta = { ...defaultMeta(req), ...metaOverrides };
             const actor =
@@ -98,7 +96,7 @@ export class HomepageController extends PuterController {
                         actor?: Parameters<typeof homepage.send>[0]['actor'];
                     }
                 ).actor ?? null;
-            await homepage.send({ req, res, actor }, meta, launch);
+            await homepage.send({ req, res, actor }, meta);
         };
 
         // -- Root + path-aliased shell routes ------------------------
@@ -111,17 +109,41 @@ export class HomepageController extends PuterController {
         router.get('/dashboard', {}, (req, res) => sendShell(req, res));
         router.get('/dashboard/', {}, (req, res) => sendShell(req, res));
 
-        router.get('/desktop', {}, (req, res) => sendShell(req, res));
-        router.get('/desktop/', {}, (req, res) => sendShell(req, res));
+        // `/desktop` used to pick the desktop interface. What follows it is
+        // the same route at the root, so it redirects there, keeping the query
+        // (`?shared=`, `?download=`); the browser carries the hash over.
+        const redirectWithoutDesktop = (
+            req: express.Request,
+            res: express.Response,
+            target: string,
+        ) => {
+            const url = req.originalUrl ?? req.url ?? '';
+            const query = url.includes('?') ? url.slice(url.indexOf('?')) : '';
+            res.redirect(302, `${target}${query}`);
+        };
+
+        router.get('/desktop', {}, (req, res) =>
+            redirectWithoutDesktop(req, res, '/'),
+        );
+        router.get('/desktop/', {}, (req, res) =>
+            redirectWithoutDesktop(req, res, '/'),
+        );
+        router.get('/desktop/app/:name', {}, (req, res) =>
+            redirectWithoutDesktop(
+                req,
+                res,
+                `/app/${encodeURIComponent(String(req.params.name))}`,
+            ),
+        );
 
         router.get('/action/*splat', {}, (req, res) => sendShell(req, res));
 
+        // `/@<user>/<path>` names something in that user's Public folder; the
+        // GUI opens it.
         router.get('/@:username', {}, (req, res) => sendShell(req, res));
+        router.get('/@:username/*splat', {}, (req, res) => sendShell(req, res));
 
         // -- /app/:name - app metadata baked into the shell ----------
-        // `/desktop/app/:name` is the same landing booted on the desktop
-        // instead of the dashboard; the GUI strips the prefix and treats the
-        // rest of the path identically, so both render the same shell.
 
         const sendAppShell = async (
             req: express.Request,
@@ -176,27 +198,10 @@ export class HomepageController extends PuterController {
             { rateLimit: APP_LANDING_LIMIT },
             sendAppShell,
         );
-        router.get(
-            '/desktop/app/:name',
-            { rateLimit: APP_LANDING_LIMIT },
-            sendAppShell,
-        );
 
-        // -- /show/* - launch explorer with the requested file path --
+        // -- /show/* - the GUI opens the path that follows ----------
 
-        router.get('/show/*splat', {}, (req, res) => {
-            const filePath = req.path.slice('/show'.length);
-            const launch: LaunchOptions = {
-                on_initialized: [
-                    {
-                        $: 'window-call',
-                        fn_name: 'launch_app',
-                        args: [{ name: 'explorer', path: filePath }],
-                    },
-                ],
-            };
-            return sendShell(req, res, {}, launch);
-        });
+        router.get('/show/*splat', {}, (req, res) => sendShell(req, res));
 
         // -- /sw.js - the service worker, scoped to the whole origin --
 

src/backend/controllers/static/StaticAssetsController.test.ts

@@ -19,8 +19,9 @@
 
 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
+import type { AddressInfo } from 'node:net';
 import path from 'node:path';
-import type { Request, Response } from 'express';
+import express, { type Request, type Response } from 'express';
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { PuterRouter } from '../../core/http/PuterRouter.js';
 import type { RouteDescriptor } from '../../core/http/types.js';
@@ -111,6 +112,7 @@ const callGetHandler = async (
     let captured: { filename?: string; root?: string } = {};
     const req = {} as Request;
     const res = {
+        setHeader: vi.fn(),
         sendFile: vi.fn((filename: string, opts: { root: string }) => {
             captured = { filename, root: opts.root };
         }),
@@ -238,6 +240,48 @@ describe('StaticAssetsController puterjs_root', () => {
     });
 });
 
+// ── push-sw.js caching ──────────────────────────────────────────────
+
+/** Serves the route's handler over real HTTP and returns the response. */
+const fetchThroughExpress = async (router: PuterRouter, routePath: string) => {
+    const route = findRoute(router, 'get', routePath);
+    if (!route) throw new Error(`No GET ${routePath} registered`);
+    const app = express();
+    app.get(routePath, (req, res, next) => route.handler(req, res, next));
+    const listener = app.listen(0);
+    await new Promise((resolve) => listener.once('listening', resolve));
+    try {
+        const { port } = listener.address() as AddressInfo;
+        return await fetch(`http://127.0.0.1:${port}${routePath}`);
+    } finally {
+        listener.close();
+    }
+};
+
+describe('StaticAssetsController push-sw.js', () => {
+    it.each([
+        ['client_libs_root', 'puter.js/push-sw.js'],
+        ['puterjs_root', 'push-sw.js'],
+    ] as const)(
+        'lets the CDN keep it for five minutes from %s',
+        async (configKey, file) => {
+            const root = makeTempDir();
+            mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
+            writeFileSync(path.join(root, file), '/* worker */');
+            const router = buildController({ [configKey]: root });
+
+            for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
+                const response = await fetchThroughExpress(router, route);
+                expect(response.status).toBe(200);
+                expect(response.headers.get('cache-control')).toBe(
+                    'public, max-age=0, s-maxage=300',
+                );
+                expect(await response.text()).toBe('/* worker */');
+            }
+        },
+    );
+});
+
 // ── gui_assets_root ─────────────────────────────────────────────────
 
 describe('StaticAssetsController gui_assets_root', () => {

src/backend/controllers/static/StaticAssetsController.ts

@@ -23,6 +23,9 @@ import path from 'node:path';
 import { PuterController } from '../types.js';
 import type { PuterRouter } from '../../core/http/PuterRouter';
 
+// Sites import push-sw.js by URL, so the CDN refetches it within five minutes.
+const PUSH_WORKER_CACHE_CONTROL = 'public, max-age=0, s-maxage=300';
+
 /**
  * Static asset routes.
  *
@@ -58,6 +61,7 @@ export class StaticAssetsController extends PuterController {
             // `/v2/push-sw.js` is the path older docs gave.
             for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
                 router.get(route, { subdomain: 'js' }, (_req, res) => {
+                    res.setHeader('Cache-Control', PUSH_WORKER_CACHE_CONTROL);
                     res.sendFile('puter.js/push-sw.js', { root });
                 });
             }
@@ -108,6 +112,10 @@ export class StaticAssetsController extends PuterController {
                 });
                 for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
                     router.get(route, { subdomain: 'js' }, (_req, res) => {
+                        res.setHeader(
+                            'Cache-Control',
+                            PUSH_WORKER_CACHE_CONTROL,
+                        );
                         res.sendFile('push-sw.js', { root: puterjsRoot });
                     });
                 }

src/backend/core/schedules.test.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { describe, expect, it } from 'vitest';
+import { isHttpError } from './http/HttpError.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    SCHEDULE_PEER_PATH,
+    schedulesUnavailable,
+} from './schedules.js';
+
+describe('schedules contract', () => {
+    it('answers an absent schedules service with a 503 events_disabled', () => {
+        const err = schedulesUnavailable();
+        expect(isHttpError(err)).toBe(true);
+        expect(err.statusCode).toBe(503);
+        expect(err.legacyCode).toBe('events_disabled');
+        // Expected on a server without schedules, so it must not page.
+        expect(err.noAlarm).toBe(true);
+    });
+
+    it('keeps the list cap and the peer path the extension relies on', () => {
+        expect(SCHEDULE_LIST_LIMIT_CAP).toBe(200);
+        expect(SCHEDULE_PEER_PATH).toBe('/broadcast/events/schedules');
+    });
+});

src/backend/core/schedules.ts

@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * What core asks of schedules. Nothing here is an implementation: an extension
+ * registers `services.schedules`, and a server without one answers every
+ * schedule route and `notify({ sendAt })` with `events_disabled`. Called by
+ * extensions; keep these signatures stable.
+ */
+
+import type { EventsDeployOutcome } from '../controllers/events/workerDeploy.js';
+import type { AppMessage } from '../services/notification/appMessage.js';
+import type { SuspendedReason } from '../stores/events/DurableSubscriptionStore.js';
+import type { DurableSubscription } from '../stores/events/types.js';
+import type { PageResult } from '../util/pagination.js';
+import type { Actor } from './actor.js';
+import { HttpError } from './http/HttpError.js';
+
+/** Why a schedule ended. */
+export type ScheduleEndReason =
+    | 'removed'
+    | 'ended'
+    | 'app_deleted'
+    | 'uninstalled'
+    | 'permission_revoked'
+    | 'holder_gone'
+    | 'app_gone'
+    | 'orphaned'
+    | 'delivered'
+    | 'dropped';
+
+/** What happened to a set of subscription rows schedules own. */
+export type ScheduleSubscriptionChange =
+    | {
+          type: 'suspended';
+          reason: SuspendedReason;
+          /** For a reason that never lifts; defaults to that reason. */
+          endReason?: ScheduleEndReason;
+      }
+    | { type: 'resumed' }
+    | { type: 'ended'; endReason: ScheduleEndReason };
+
+/** A schedule that keeps running, one that runs once, or a delayed notification. */
+export type ScheduleViewKind = 'recurring' | 'once' | 'deferred';
+
+/**
+ * A schedule as its holder sees it. Times are epoch ms. `context` values are
+ * never returned, as for a persistent subscription: only its keys and a hash.
+ */
+export interface ScheduleView {
+    scheduleId: string;
+    kind: ScheduleViewKind;
+    appUid: string;
+    /** `null` once ended: the handler binding goes with the schedule. */
+    handlerName: string | null;
+    /** Set only when the handler lives in the holder's own scope. */
+    handlerScope?: 'user';
+    contextKeys: string[] | null;
+    contextHash: string | null;
+    /** Interval of an `every` schedule. */
+    every: number | null;
+    cron: string | null;
+    tz: string;
+    /** First allowed run; a one-shot's only run. */
+    startAt: number;
+    /** Last allowed run, inclusive. */
+    endsAt: number | null;
+    /** `sending` once a delayed send has started; it can't be cancelled then. */
+    state: 'active' | 'suspended' | 'sending' | 'ended';
+    suspendedReason: string | null;
+    endReason: string | null;
+    endedAt: number | null;
+    /**
+     * The slot it runs for next; `null` while suspended or sending, and once
+     * ended.
+     */
+    nextRunAt: number | null;
+    createdAt: number;
+}
+
+export interface ScheduleListRequest {
+    limit: number;
+    cursor?: string;
+    includeTotal: boolean;
+}
+
+/** The caps a holder's schedules are held to, from their plan. */
+export interface ScheduleQuota {
+    recurringPerUser: number;
+    recurringPerApp: number;
+    oneShotsPerUser: number;
+    oneShotsPerApp: number;
+    /** Worst-case runs per hour × 1000, summed over recurring rows. */
+    budgetPerUser: number;
+    budgetPerApp: number;
+}
+
+/** A caller cleared to supply `sendAt`, and when it asked for. */
+export interface DeferredSendPlan {
+    sendAt: number;
+    /** Due within 5 seconds, or past: sent now rather than stored. */
+    immediate: boolean;
+    holderUserId: number;
+    appUid: string;
+    handlerDepth: number;
+    quota: ScheduleQuota;
+}
+
+/** What a delayed app message stores until it is sent. */
+export interface DeferredAppMessage {
+    message: AppMessage;
+    /** A KV share handle naming the recipient. */
+    to?: string;
+}
+
+/**
+ * A condition on one row of another table that statements embed as `EXISTS
+ * (SELECT 1 FROM <table> WHERE <where>)`, binding `values`.
+ */
+export interface SqlGuard {
+    table: string;
+    where: string;
+    values: unknown[];
+}
+
+/** Ceiling of a schedule listing page. */
+export const SCHEDULE_LIST_LIMIT_CAP = 200;
+
+/** The signed peer route a region's schedule changes are sent to. */
+export const SCHEDULE_PEER_PATH = '/broadcast/events/schedules';
+
+export const schedulesUnavailable = (): HttpError =>
+    new HttpError(503, 'Schedules are not available on this server', {
+        legacyCode: 'events_disabled',
+        // The designed answer of a server without schedules, not a fault.
+        noAlarm: true,
+    });
+
+export interface ScheduleServiceContract {
+    /**
+     * Keep schedules in step with a change to the subscription rows they own.
+     * Never throws: the subscription change has already happened.
+     */
+    onSubscriptionsChanged(
+        rows: readonly DurableSubscription[],
+        change: ScheduleSubscriptionChange,
+    ): Promise<void>;
+    createFor(
+        actor: Actor,
+        body: Record<string, unknown>,
+    ): Promise<ScheduleView>;
+    list(
+        actor: Actor,
+        request: ScheduleListRequest,
+    ): Promise<PageResult<ScheduleView>>;
+    get(actor: Actor, scheduleId: unknown): Promise<ScheduleView>;
+    remove(actor: Actor, scheduleId: unknown): Promise<void>;
+    receivePeerChanges(body: unknown): Promise<{ applied: number }>;
+    /** Checks a caller supplying `sendAt`; `events_disabled` when off. */
+    prepareSendAt(actor: Actor, sendAt: unknown): Promise<DeferredSendPlan>;
+    createDeferredAppMessage(
+        plan: DeferredSendPlan,
+        message: DeferredAppMessage,
+    ): Promise<{ scheduleId: string; sendAt: number }>;
+    /**
+     * Runs an events worker deploy under the schedules' deploy gate; `capped`
+     * for a schedule run's. A passthrough when schedules are off.
+     */
+    gateDeploy?(
+        script: string,
+        options: { capped: boolean },
+        deploy: () => Promise<EventsDeployOutcome>,
+    ): Promise<EventsDeployOutcome>;
+}

src/backend/drivers/apps/AppDriver.js

@@ -31,7 +31,7 @@ import {
     normalizeRawBase64ImageString,
     validateIconDataUrl,
 } from '../../util/appIcon.js';
-import { toAppReadView } from '../../util/appView.js';
+import { isOriginBootstrapApp, toAppReadView } from '../../util/appView.js';
 import { awaitLater } from '../../util/concurrency.js';
 import { isUniqueViolation } from '../../util/dbError.js';
 import {
@@ -1088,22 +1088,6 @@ export class AppDriver extends PuterDriver {
         }
     }
 
-    /**
-     * Origin-bootstrap detection: rows auto-created when an unknown origin
-     * first needed an app row (no human-supplied metadata). Marker is `name ===
-     * uid && title === uid` and a description starting with "App created from
-     * origin ". Only these rows are eligible for same-owner merging — refusing
-     * to merge arbitrary same-owner apps prevents accidental data loss.
-     */
-    #isOriginBootstrapApp(app) {
-        if (!app || typeof app !== 'object') return false;
-        if (typeof app.uid !== 'string' || !app.uid) return false;
-        if (app.name !== app.uid) return false;
-        if (app.title !== app.uid) return false;
-        if (typeof app.description !== 'string') return false;
-        return app.description.startsWith('App created from origin ');
-    }
-
     // -- Canonical-uid alias kvstore pair -----------------------------
     //
     // After a merge, the source app uid → canonical uid mapping is
@@ -1266,7 +1250,7 @@ export class AppDriver extends PuterDriver {
         if (
             Number.isInteger(conflictOwnerUserId) &&
             conflictOwnerUserId === user.id &&
-            !this.#isOriginBootstrapApp(appToJoin)
+            !isOriginBootstrapApp(appToJoin)
         ) {
             // Prevent merging arbitrary same-owner apps; only allow the
             // auto-created origin bootstrap row to be absorbed.

src/backend/drivers/notification/NotificationDriver.sendAt.test.ts

@@ -0,0 +1,493 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * `notify({ sendAt })` without and with `services.schedules`, and the deferred
+ * mailbox write an extension runs inside its own transaction, fenced on a guard
+ * in a table of its own (a test table here).
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { runWithContext } from '../../core/context.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    DeferredSendPlan,
+    ScheduleServiceContract,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { EVENTS_BACKGROUND_PERMISSION } from '../../services/events/authorization.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import type {
+    DeferredWriteTx,
+    NotificationDriver,
+} from './NotificationDriver.js';
+
+vi.setConfig({ testTimeout: 120_000 });
+
+const BOOT_TIMEOUT_MS = 480_000;
+const TEST_TABLE = 'deferred_send_test';
+
+let server: PuterServer;
+let holder: Actor;
+let app: { id: number; uid: string };
+let saved: unknown;
+let nextId = 1;
+
+const db = () => server.clients.db;
+const driver = () =>
+    server.drivers.notifications as unknown as NotificationDriver;
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+const asApp = (): Actor => makeActor({ user: holder.user, app });
+
+const notify = (fields: Record<string, unknown>): Promise<unknown> =>
+    runWithContext({ actor: asApp() }, () =>
+        driver().create({
+            object: { value: { type: 'app.message', title: 'Hi', ...fields } },
+        }),
+    );
+
+const codeOf = async (run: Promise<unknown>): Promise<string | undefined> => {
+    try {
+        await run;
+    } catch (err) {
+        if (isHttpError(err)) return err.legacyCode;
+        throw err;
+    }
+    return undefined;
+};
+
+const mailbox = () =>
+    db().read(
+        'SELECT * FROM `notification` WHERE `user_id` = ? AND `app_uid` = ? ' +
+            'ORDER BY `id`',
+        [holder.user!.id, app.uid],
+    );
+
+/** A row's value: JSON text on sqlite, parsed on the other engines. */
+const valueOf = (row: Record<string, unknown>) =>
+    typeof row.value === 'string' ? JSON.parse(row.value) : row.value;
+
+/** A send claimed under `token`, as an extension's own row would be. */
+const claimed = async (token: string): Promise<number> => {
+    const id = nextId++;
+    await db().write(
+        `INSERT INTO \`${TEST_TABLE}\` (\`id\`, \`token\`, \`state\`) VALUES (?, ?, 'sending')`,
+        [id, token],
+    );
+    return id;
+};
+
+/** The transaction an extension wraps the write in: its row, then the mailbox. */
+const txFor = (id: number, token: string): DeferredWriteTx => ({
+    guard: {
+        table: TEST_TABLE,
+        where: "`id` = ? AND `token` = ? AND `state` = 'delivered'",
+        values: [id, token],
+    },
+    async commit(entries: readonly BatchEntry[]) {
+        await db().batchWrite([
+            {
+                statement:
+                    `UPDATE \`${TEST_TABLE}\` SET \`state\` = 'delivered' ` +
+                    "WHERE `id` = ? AND `token` = ? AND `state` = 'sending'",
+                values: [id, token],
+            },
+            ...entries,
+        ]);
+        const [row] = await db().pread(
+            `SELECT \`state\`, \`token\` FROM \`${TEST_TABLE}\` WHERE \`id\` = ?`,
+            [id],
+        );
+        return row?.state === 'delivered' && row.token === token;
+    },
+});
+
+const write = (
+    sendKey: string,
+    tx: DeferredWriteTx,
+    over: {
+        payload?: string | null;
+        appUid?: string;
+        signal?: AbortSignal;
+    } = {},
+) =>
+    driver().deferredAppMessages.write(
+        {
+            holderUserId: holder.user!.id,
+            appUid: over.appUid ?? app.uid,
+            payload:
+                over.payload === undefined
+                    ? JSON.stringify({ message: { title: 'Later' } })
+                    : over.payload,
+            sendKey,
+            signal: over.signal ?? new AbortController().signal,
+        },
+        tx,
+    );
+
+const fakeSchedules = (plan: Partial<DeferredSendPlan>) =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(),
+        list: vi.fn(),
+        get: vi.fn(),
+        remove: vi.fn(),
+        receivePeerChanges: vi.fn(),
+        prepareSendAt: vi.fn(async (): Promise<DeferredSendPlan> => ({
+            sendAt: 0,
+            immediate: false,
+            holderUserId: holder.user!.id,
+            appUid: app.uid,
+            handlerDepth: 0,
+            quota: {
+                recurringPerUser: 1,
+                recurringPerApp: 1,
+                oneShotsPerUser: 1,
+                oneShotsPerApp: 1,
+                budgetPerUser: 1,
+                budgetPerApp: 1,
+            },
+            ...plan,
+        })),
+        createDeferredAppMessage: vi.fn(async (p: DeferredSendPlan) => ({
+            scheduleId: 'sched-1',
+            sendAt: p.sendAt,
+        })),
+    }) satisfies ScheduleServiceContract;
+
+beforeAll(async () => {
+    server = await setupTestServer({
+        events: { enabled: true },
+    } as IConfig);
+    const username = `sendat-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+        requires_email_confirmation: false,
+    });
+    const row = (await server.stores.user.getById(user.id))!;
+    holder = makeActor({
+        user: {
+            id: row.id,
+            uuid: row.uuid,
+            username: row.username,
+            email: row.email ?? null,
+            email_confirmed: true,
+        } as Actor['user'],
+    });
+    const name = `sendat-app-${Math.random().toString(36).slice(2, 10)}`;
+    app = (await server.stores.app.create(
+        { name, title: name, index_url: `https://${name}.test/` },
+        { ownerUserId: row.id },
+    )) as { id: number; uid: string };
+    await server.services.permission.grantUserAppPermission(
+        holder,
+        app.uid,
+        EVENTS_BACKGROUND_PERMISSION,
+    );
+    await db().write(
+        `CREATE TABLE \`${TEST_TABLE}\` (\`id\` INTEGER PRIMARY KEY, ` +
+            '`token` VARCHAR(64) NOT NULL, `state` VARCHAR(16) NOT NULL)',
+        [],
+    );
+    saved = slot().schedules;
+}, BOOT_TIMEOUT_MS);
+
+beforeEach(async () => {
+    await db().write('DELETE FROM `notification` WHERE `app_uid` = ?', [
+        app.uid,
+    ]);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+    vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+describe('notify({ sendAt }) without services.schedules', () => {
+    it('is refused events_disabled, even when due now, storing nothing', async () => {
+        setSchedules(undefined);
+        expect(await codeOf(notify({ sendAt: Date.now() }))).toBe(
+            'events_disabled',
+        );
+        expect(await codeOf(notify({ sendAt: Date.now() + 3_600_000 }))).toBe(
+            'events_disabled',
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends as before without sendAt', async () => {
+        setSchedules(undefined);
+        const sent = (await notify({ text: 'now' })) as { uid: string };
+        expect(sent.uid).toBeTruthy();
+        expect(await mailbox()).toHaveLength(1);
+    });
+});
+
+describe('notify({ sendAt }) with services.schedules', () => {
+    it('hands a later send to createDeferredAppMessage and answers its id', async () => {
+        const at = Date.now() + 3_600_000;
+        const schedules = fakeSchedules({ sendAt: at });
+        setSchedules(schedules);
+
+        await expect(notify({ sendAt: at, text: 'later' })).resolves.toEqual({
+            scheduled: { scheduleId: 'sched-1', sendAt: at },
+        });
+        expect(schedules.prepareSendAt).toHaveBeenCalledWith(
+            expect.objectContaining({ user: holder.user }),
+            at,
+        );
+        expect(schedules.createDeferredAppMessage).toHaveBeenCalledWith(
+            expect.objectContaining({ sendAt: at }),
+            {
+                message: expect.objectContaining({
+                    title: 'Hi',
+                    text: 'later',
+                }),
+            },
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends at once when the plan says it is due', async () => {
+        const schedules = fakeSchedules({ immediate: true });
+        setSchedules(schedules);
+
+        const sent = (await notify({ sendAt: Date.now() })) as {
+            uid: string;
+        };
+        expect(sent.uid).toBeTruthy();
+        expect(schedules.createDeferredAppMessage).not.toHaveBeenCalled();
+        expect(await mailbox()).toHaveLength(1);
+    });
+
+    it('passes a refusal from prepareSendAt through', async () => {
+        const schedules = fakeSchedules({});
+        schedules.prepareSendAt.mockRejectedValueOnce(
+            Object.assign(new Error('scoped'), {}),
+        );
+        setSchedules(schedules);
+        await expect(notify({ sendAt: Date.now() })).rejects.toThrow('scoped');
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+describe('deferredAppMessages.write', () => {
+    it('writes one row keyed by the send, in the caller`s transaction', async () => {
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(write(key, txFor(id, token))).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: key, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
+    });
+
+    it('writes nothing when the guard no longer holds: another claim took it', async () => {
+        const id = await claimed(uuidv4());
+        const key = uuidv4();
+
+        await expect(write(key, txFor(id, uuidv4()))).resolves.toBe('lost');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('replaces the open row of its group, which takes the key', async () => {
+        const open = (await notify({ group: 'g', text: 'first' })) as {
+            uid: string;
+        };
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(
+            write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g' },
+                }),
+            }),
+        ).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: open.uid, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
+    });
+
+    it('drops a payload it can`t read, and a sender whose app is gone', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        await expect(
+            write(uuidv4(), txFor(id, token), { payload: '{"nope":1}' }),
+        ).resolves.toBe('dropped');
+        await expect(
+            write(uuidv4(), txFor(id, token), { appUid: `app-${uuidv4()}` }),
+        ).resolves.toBe('dropped');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('writes nothing once the caller`s claim is lost', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        const lost = new AbortController();
+        lost.abort();
+
+        await expect(
+            write(uuidv4(), txFor(id, token), { signal: lost.signal }),
+        ).rejects.toThrow();
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+/**
+ * A second claim taking a send over while the first one's transaction is open.
+ * Needs two writers at once with row locks; sqlite serialises writers, so this
+ * runs on postgres only.
+ */
+describe.runIf(process.env.PUTER_TEST_DB_ENGINE === 'postgres')(
+    'a deferred write racing another writer',
+    () => {
+        /**
+         * Hold the first statement matching `match` on its connection, before
+         * or after it runs, until released.
+         */
+        const holdStatement = (match: RegExp, when: 'before' | 'after') => {
+            const client = db() as unknown as {
+                query: (
+                    target: unknown,
+                    sql: string,
+                    params?: unknown[],
+                ) => Promise<unknown>;
+            };
+            const original = client.query.bind(client);
+            let reached!: () => void;
+            let release!: () => void;
+            const atBarrier = new Promise<void>((r) => (reached = r));
+            const released = new Promise<void>((r) => (release = r));
+            let held = false;
+            vi.spyOn(client as never, 'query' as never).mockImplementation(
+                (async (target: unknown, sql: string, params?: unknown[]) => {
+                    if (held || !match.test(sql))
+                        return original(target, sql, params);
+                    held = true;
+                    if (when === 'before') {
+                        reached();
+                        await released;
+                        return original(target, sql, params);
+                    }
+                    const result = await original(target, sql, params);
+                    reached();
+                    await released;
+                    return result;
+                }) as never,
+            );
+            return { atBarrier, release };
+        };
+
+        const DELIVERED = /SET `state` = 'delivered'/;
+        const REPLACE = /`replaced_at` = \?, `send_key` = \? WHERE/;
+
+        /** Whether `promise` is still pending after `ms`. */
+        const stillPending = async (promise: Promise<unknown>, ms: number) => {
+            let settled = false;
+            const watched = (async () => {
+                try {
+                    await promise;
+                } finally {
+                    settled = true;
+                }
+            })();
+            await new Promise((r) => setTimeout(r, ms));
+            void watched;
+            return !settled;
+        };
+
+        it('a transaction that opened before a takeover writes nothing', async () => {
+            const stale = uuidv4();
+            const id = await claimed(stale);
+            const key = uuidv4();
+            const gate = holdStatement(DELIVERED, 'before');
+            const first = write(key, txFor(id, stale));
+            await gate.atBarrier;
+
+            await db().write(
+                `UPDATE \`${TEST_TABLE}\` SET \`token\` = ? WHERE \`id\` = ?`,
+                [uuidv4(), id],
+            );
+            gate.release();
+            await expect(first).resolves.toBe('lost');
+            expect(
+                await server.stores.notification.getBySendKey(key),
+            ).toBeNull();
+            expect(await mailbox()).toEqual([]);
+        });
+
+        it('an immediate replacement waiting on the open transaction lands after it, without the key', async () => {
+            const open = (await notify({ group: 'g', text: 'first' })) as {
+                uid: string;
+            };
+            const token = uuidv4();
+            const id = await claimed(token);
+            const key = uuidv4();
+            const gate = holdStatement(REPLACE, 'after');
+            const deferred = write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g', text: 'later' },
+                }),
+            });
+            await gate.atBarrier;
+
+            const immediate = notify({ group: 'g', text: 'newer' });
+            // Past the group lock's wait, then blocked on the row.
+            expect(await stillPending(immediate, 2_500)).toBe(true);
+            gate.release();
+            await expect(deferred).resolves.toBe('delivered');
+            await immediate;
+
+            const rows = await mailbox();
+            expect(rows).toHaveLength(1);
+            expect(rows[0]).toMatchObject({ uid: open.uid, send_key: null });
+            expect(valueOf(rows[0]).text).toBe('newer');
+        });
+    },
+);

src/backend/drivers/notification/NotificationDriver.ts

@@ -51,6 +51,14 @@ import {
     resolveNotificationWrite,
     type NotificationTypeName,
 } from '../../services/notification/notificationTypes.js';
+import {
+    schedulesUnavailable,
+    type DeferredAppMessage,
+    type DeferredSendPlan,
+    type ScheduleServiceContract,
+    type SqlGuard,
+} from '../../core/schedules.js';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
 import { withRedisLock } from '../../util/redisLock.js';
 import { PuterDriver } from '../types.js';
 import type { Actor } from '../../core/actor.js';
@@ -67,6 +75,44 @@ const GROUP_LOCK_RETRY_MS = 25;
 
 type AppMessageWindow = 'minute' | 'day';
 
+export type { SqlGuard };
+
+/** What a delayed app message's fire-time write needs from its row. */
+export interface DeferredWriteInput {
+    holderUserId: number;
+    appUid: string;
+    /** The stored {@link DeferredAppMessage}, as JSON text. */
+    payload: string | null;
+    /** The slot's key: the written row's uid or `send_key`, and its `sendId`. */
+    sendKey: string;
+    /** Aborted when the caller's claim is lost. */
+    signal: AbortSignal;
+}
+
+/**
+ * The caller's transaction the mailbox write commits in. Atomicity is the
+ * caller's to keep; the type can't enforce it.
+ */
+export interface DeferredWriteTx {
+    /**
+     * What every mailbox statement is fenced on: the caller's row marked
+     * delivered by this claim.
+     */
+    guard: SqlGuard;
+    /**
+     * Must be the whole delivery in one step: the caller's guarded transition
+     * of its row to delivered and `entries`, in that order, in one
+     * `batchWrite`, then a read of the row on the primary. True only when that
+     * read shows this claim holding the row as delivered. Splitting the
+     * transition from `entries` lets one slot write two rows.
+     */
+    commit(entries: readonly BatchEntry[]): Promise<boolean>;
+}
+
+/** How a delayed send's write stood when it stopped. */
+export type DeferredWriteOutcome =
+    'delivered' | 'dropped' | 'lost' | 'senderSuspended';
+
 /** The mailbox slice a call named, in the terms the query takes it. */
 interface MailboxScope {
     audiences: readonly string[];
@@ -124,6 +170,21 @@ const parseNotifyTarget = (to: unknown): string => {
     return target.handle;
 };
 
+/** A stored delayed message, or `null` for one that can't be read back. */
+const parseDelayedAppMessage = (
+    payload: string | null,
+): DeferredAppMessage | null => {
+    try {
+        const stored = JSON.parse(payload ?? 'null') as DeferredAppMessage;
+        if (typeof stored?.message?.title !== 'string') return null;
+        if (stored.to !== undefined && typeof stored.to !== 'string')
+            return null;
+        return stored;
+    } catch {
+        return null;
+    }
+};
+
 /** Whether a row belongs to the slice, as the same SQL scope selected it. */
 const inScope = (
     row: Record<string, unknown>,
@@ -354,7 +415,7 @@ export class NotificationDriver extends PuterDriver {
         object: Record<string, unknown>,
         value: Record<string, unknown>,
     ): Promise<unknown> {
-        const { type, to, ...fields } = value;
+        const { type, to, sendAt, ...fields } = value;
         if (type !== APP_MESSAGE_TYPE) {
             throw typeNotAllowed(`Apps can create only ${APP_MESSAGE_TYPE}`);
         }
@@ -370,6 +431,12 @@ export class NotificationDriver extends PuterDriver {
         const handle = to === undefined ? undefined : parseNotifyTarget(to);
         const message = parseAppMessage(fields);
         assertValueFits(value);
+        const timing =
+            sendAt === undefined || sendAt === null
+                ? null
+                : await this.#schedules().prepareSendAt(actor, sendAt);
+        if (timing && !timing.immediate)
+            return this.#scheduleAppMessage(actor, message, handle, timing);
 
         if (handle === undefined) {
             const uid = await this.#sendAppMessage(
@@ -410,12 +477,23 @@ export class NotificationDriver extends PuterDriver {
         appUid: string,
         message: AppMessage,
     ): Promise<string> {
-        const send = () =>
-            this.#replaceOrWriteAppMessage(recipientId, appUid, message);
+        return this.#oneSendPerGroup(recipientId, appUid, message, () =>
+            this.#replaceOrWriteAppMessage(recipientId, appUid, message),
+        );
+    }
+
+    /**
+     * One send per group at a time, so a burst into an empty group writes one
+     * row that the rest replace. A lock that is busy past the wait or
+     * unreachable still sends: the race costs a second row and its day slot.
+     */
+    #oneSendPerGroup<T>(
+        recipientId: number,
+        appUid: string,
+        message: AppMessage,
+        send: () => Promise<T>,
+    ): Promise<T> {
         if (!message.group) return send();
-        // One send per group at a time, so a burst into an empty group writes
-        // one row that the rest replace. A lock that is busy past the wait or
-        // unreachable still sends: the race costs a second row and its day slot.
         return withRedisLock(
             this.clients.redis,
             `notif:app-message:group:${recipientId}:${appUid}:${message.group}`,
@@ -430,6 +508,163 @@ export class NotificationDriver extends PuterDriver {
         );
     }
 
+    // -- Delayed sends -----------------------------------------------
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /**
+     * Store an app's message to go out at `sendAt`. A recipient named through a
+     * share is checked now without spending its budget; every limit is spent
+     * when it goes, and the share is checked again then.
+     */
+    async #scheduleAppMessage(
+        actor: UserActor,
+        message: AppMessage,
+        handle: string | undefined,
+        plan: DeferredSendPlan,
+    ): Promise<{ scheduled: { scheduleId: string; sendAt: number } }> {
+        if (handle !== undefined)
+            await this.services.kvShare.resolveNotifyRecipient(actor, handle, {
+                spendBudget: false,
+            });
+        return {
+            scheduled: await this.#schedules().createDeferredAppMessage(
+                plan,
+                handle === undefined ? { message } : { message, to: handle },
+            ),
+        };
+    }
+
+    /**
+     * For extensions. A field, not a method, so `/drivers/call` can't reach it.
+     */
+    readonly deferredAppMessages = {
+        write: (input: DeferredWriteInput, tx: DeferredWriteTx) =>
+            this.#writeDeferredAppMessage(input, tx),
+    };
+
+    /**
+     * Check the sender and recipient as they stand now, spend the recipient's
+     * limits, and write the mailbox row through `tx`, fenced on its guard.
+     * Refusals are the same `HttpError`s an immediate send throws.
+     */
+    async #writeDeferredAppMessage(
+        input: DeferredWriteInput,
+        tx: DeferredWriteTx,
+    ): Promise<DeferredWriteOutcome> {
+        const { holderUserId, sendKey, signal } = input;
+        const sender = await this.services.events.resolveScheduleActor(
+            holderUserId,
+            input.appUid,
+        );
+        if (!sender) {
+            // A lagging replica can miss an app the primary has: retried.
+            const there =
+                (await this.stores.app.uidExistsOnPrimary(input.appUid)) &&
+                (await this.stores.user.getById(holderUserId)) !== null;
+            if (there) throw new Error('the sender could not be resolved');
+            return 'dropped';
+        }
+        if (sender.user?.suspended) return 'senderSuspended';
+        // Uninstalled since, and the cancel at uninstall missed it.
+        const appId = sender.effectiveApp?.id;
+        if (
+            typeof appId !== 'number' ||
+            !(await this.stores.permission.hasAnyUserAppPerm(
+                holderUserId,
+                appId,
+            ))
+        )
+            return 'dropped';
+        const stored = parseDelayedAppMessage(input.payload);
+        if (!stored) return 'dropped';
+        const { message } = stored;
+
+        let recipientId = holderUserId;
+        let appUid = input.appUid;
+        if (stored.to !== undefined) {
+            const recipient =
+                await this.services.kvShare.resolveNotifyRecipient(
+                    sender,
+                    stored.to,
+                );
+            recipientId = recipient.recipientUserId;
+            appUid = recipient.appUid;
+        }
+
+        return this.#oneSendPerGroup(recipientId, appUid, message, async () => {
+            // Primary, as for an immediate send.
+            const open = message.group
+                ? await this.stores.notification.getUnseenByGroup(
+                      recipientId,
+                      appUid,
+                      message.group,
+                      { primary: true },
+                  )
+                : null;
+            await this.#spendAppMessageLimits(
+                recipientId,
+                appUid,
+                open ? ['minute'] : ['minute', 'day'],
+            );
+            message.app = appMessageSender(
+                appUid,
+                await this.stores.app.getByUid(appUid),
+            );
+            // Another attempt may hold the lease now; hand the claim back.
+            if (signal.aborted) throw new Error('delayed send lost its lease');
+
+            const registered = resolveNotificationWrite(
+                APP_MESSAGE_TYPE,
+                appUid,
+            );
+            const delivered = await tx.commit(
+                this.stores.notification.sendEntries(
+                    {
+                        sendKey,
+                        userId: recipientId,
+                        value: { ...message, type: registered.type },
+                        type: registered.type,
+                        audience: registered.audience,
+                        appUid,
+                        group: message.group ?? null,
+                        replaceUid: open ? String(open.uid) : null,
+                        at: Date.now(),
+                    },
+                    tx.guard,
+                ),
+            );
+            if (!delivered) return 'lost';
+            if (open)
+                await this.#spendDayIfFresh(recipientId, appUid, open, sendKey);
+            return 'delivered';
+        });
+    }
+
+    /**
+     * A replacement that found its row shown since the lookup wrote a new one,
+     * which costs a day slot. Already written, so a full day is not a refusal.
+     */
+    async #spendDayIfFresh(
+        recipientId: number,
+        appUid: string,
+        open: Record<string, unknown>,
+        sendKey: string,
+    ): Promise<void> {
+        try {
+            const written =
+                await this.stores.notification.getBySendKey(sendKey);
+            if (written && written.uid !== open.uid)
+                await this.#spendAppMessageLimits(recipientId, appUid, ['day']);
+        } catch {
+            // The row is written; a slot not spent is the only loss.
+        }
+    }
+
     async #replaceOrWriteAppMessage(
         recipientId: number,
         appUid: string,

src/backend/drivers/util/pipeStreamResult.test.ts

@@ -129,6 +129,27 @@ describe('pipeStreamResult', () => {
         expect(warn).not.toHaveBeenCalled();
     });
 
+    it('adds at most two close listeners to the response', async () => {
+        // The response already carries close listeners from metering,
+        // compression and tracing; Node warns past ten.
+        let added = -1;
+        server = createServer((_req, res) => {
+            const before = res.listenerCount('close');
+            pipeStreamResult(Readable.from(['a', 'b']), res, 'test.run');
+            added = res.listenerCount('close') - before;
+        });
+        await new Promise<void>((resolve) => server!.listen(0, resolve));
+        const { port } = server.address() as AddressInfo;
+        await new Promise<void>((resolve) =>
+            get(`http://127.0.0.1:${port}/`, (res) => {
+                res.resume();
+                res.on('end', () => resolve());
+            }),
+        );
+        expect(added).toBeGreaterThan(0);
+        expect(added).toBeLessThanOrEqual(2);
+    });
+
     it('drops the source, without throwing, when the client left before the result was ready', async () => {
         const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
         const source = Readable.from(['late']);

src/backend/drivers/util/pipeStreamResult.ts

@@ -17,7 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import { pipeline, type Readable } from 'node:stream';
+import type { Readable } from 'node:stream';
 
 // Only what's used here, so callers on another @types/node (or express's
 // Response) still fit.
@@ -32,24 +32,32 @@ type ResponseSink = NodeJS.WritableStream & {
  * mid-body destroys the response, so the client sees an aborted transfer rather
  * than a clean end, and the failure is logged once instead of surfacing as an
  * unhandled stream error. A client that hangs up stops the source.
+ *
+ * `pipe` rather than `pipeline`: pipeline puts seven close listeners on the
+ * response, and with the ones metering, compression and tracing add, every
+ * streamed response tripped Node's ten-listener warning.
  */
 export function pipeStreamResult(
     stream: Readable,
     res: ResponseSink,
     label: string,
 ): void {
-    // The caller hung up while the driver ran; `pipeline` throws on a closed
-    // destination, so drop the source instead.
+    // The caller hung up while the driver ran: nothing will read the source.
     if (res.closed || res.destroyed) {
         stream.destroy();
         return;
     }
-    pipeline(stream, res, (err) => {
-        if (!err) return;
+    stream.once('error', (err) => {
+        stream.unpipe(res);
         if (!res.destroyed) res.destroy();
-        // The client hanging up is not a failure of ours.
-        if ((err as { code?: string }).code === 'ERR_STREAM_PREMATURE_CLOSE')
-            return;
         console.warn(`[drivers] ${label} stream failed mid-response:`, err);
     });
+    // A response that closes before the source ended is a client hanging up.
+    res.once('close', () => {
+        if (stream.readableEnded) return;
+        stream.unpipe(res);
+        stream.destroy();
+    });
+    res.once('error', () => stream.destroy());
+    stream.pipe(res);
 }

src/backend/drivers/workers/WorkerDriver.cloudflare.test.ts

@@ -754,6 +754,50 @@ describe('WorkerDriver.create with a configured deploy backend', () => {
             expect(putCalls()).toHaveLength(1);
         });
 
+        const uploadedMetadata = () => {
+            const [, init] = putCalls().at(-1)!;
+            const form = (init as RequestInit).body as FormData;
+            return JSON.parse(form.get('metadata') as string);
+        };
+
+        it('sends limits.cpu_ms in the upload metadata when cpuMs is set', async () => {
+            const { user, actor } = await makeUser();
+            const path = `/${user.username}/cpu.worker.js`;
+            await writeSource(actor, user.id, path, 'export default {}');
+            const scriptName = `dt-cpu-${user.username}`;
+            await inCtx(actor, () =>
+                target.create({
+                    appId: '',
+                    workerName: scriptName,
+                    filePath: path,
+                    [INTERNAL_DEPLOY_TARGET]: { scriptName, cpuMs: 50 },
+                }),
+            );
+            expect(uploadedMetadata().limits).toEqual({ cpu_ms: 50 });
+        });
+
+        it('sends no limits key without cpuMs, and none on an ordinary create', async () => {
+            const { user, actor } = await makeUser();
+            const path = `/${user.username}/nocpu.worker.js`;
+            await writeSource(actor, user.id, path, 'export default {}');
+            const scriptName = `dt-nocpu-${user.username}`;
+            await deployTo(actor, path, scriptName);
+            expect(uploadedMetadata()).not.toHaveProperty('limits');
+
+            const name = `plain-${user.username}`;
+            await inCtx(actor, () =>
+                target.create({ appId: '', workerName: name, filePath: path }),
+            );
+            const metadata = uploadedMetadata();
+            expect(metadata).not.toHaveProperty('limits');
+            expect(Object.keys(metadata)).toEqual([
+                'body_part',
+                'compatibility_flags',
+                'compatibility_date',
+                'bindings',
+            ]);
+        });
+
         it('refuses a file another user owns, even with manage on it', async () => {
             const owner = await makeUser();
             const deployer = await makeUser();

src/backend/drivers/workers/WorkerDriver.test.ts

@@ -3,6 +3,7 @@
 //  however the interactions with the Puter API are
 import {
     afterAll,
+    afterEach,
     beforeAll,
     beforeEach,
     describe,
@@ -13,6 +14,7 @@ import {
 import { Actor, makeActor as resolveActor } from '../../core/actor.js';
 import { runWithContext } from '../../core/context.js';
 import { PuterServer } from '../../server.js';
+import { DEFAULT_FREE_SUBSCRIPTION } from '../../services/metering/consts.js';
 import {
     setupPuterTestEnv,
     setupTestServer,
@@ -797,6 +799,174 @@ describe('WorkerDriver', () => {
         });
     });
 
+    describe('rented-unit cap', () => {
+        const FREE_CAP = 16;
+        const PAID_CAP = 100;
+
+        // The cap sits after the deploy-credentials gate, so the driver needs
+        // credentials to reach it; a create under the cap then fails on the
+        // missing source file, which is what tells it past the cap check.
+        let originalWorkersConfig: unknown;
+        let originalEvents: unknown;
+        beforeAll(() => {
+            const driverConfig = (
+                target as unknown as { config: Record<string, unknown> }
+            ).config;
+            originalEvents = driverConfig.events;
+            driverConfig.events = { userHandlers: true };
+            originalWorkersConfig = driverConfig.workers;
+            driverConfig.workers = {
+                XAUTHKEY: 'test-key',
+                ACCOUNTID: 'test-account',
+            };
+        });
+        afterAll(() => {
+            const driverConfig = (
+                target as unknown as { config: Record<string, unknown> }
+            ).config;
+            driverConfig.workers = originalWorkersConfig;
+            driverConfig.events = originalEvents;
+        });
+
+        const holding = (
+            plan: string,
+            held: { workers?: number; scopes?: number; owned?: number },
+        ) => {
+            vi.spyOn(
+                server.services.metering,
+                'getActorSubscription',
+            ).mockResolvedValue({ id: plan } as never);
+            vi.spyOn(
+                server.stores.subdomain,
+                'countByUserIdAndPrefix',
+            ).mockResolvedValue(held.workers ?? 0);
+            vi.spyOn(
+                server.stores.eventHandler,
+                'countUserScopesWithHandlersForUser',
+            ).mockResolvedValue(held.scopes ?? 0);
+            vi.spyOn(
+                server.stores.eventHandler,
+                'countAppsWithHandlersForOwner',
+            ).mockResolvedValue(held.owned ?? 0);
+        };
+        afterEach(() => {
+            vi.restoreAllMocks();
+        });
+
+        const createWorker = () =>
+            inCtx(() =>
+                target.create({
+                    appId: 'test-app',
+                    workerName: 'capped',
+                    filePath: '/test.js',
+                }),
+            );
+
+        it('keeps the flat 100-worker cap over ordinary workers with user scopes off', async () => {
+            const driverConfig = (
+                target as unknown as { config: Record<string, unknown> }
+            ).config;
+            driverConfig.events = { userHandlers: false };
+            try {
+                // 16 workers is fine here, and scopes and owned apps don't count.
+                holding(DEFAULT_FREE_SUBSCRIPTION, {
+                    workers: 16,
+                    scopes: 50,
+                    owned: 50,
+                });
+                const list = vi
+                    .spyOn(server.stores.subdomain, 'listByUserIdAndPrefix')
+                    .mockResolvedValue(
+                        Array.from({ length: 16 }, () => ({})) as never,
+                    );
+                await expect(createWorker()).rejects.not.toMatchObject({
+                    statusCode: 403,
+                });
+
+                list.mockResolvedValue(
+                    Array.from({ length: 100 }, () => ({})) as never,
+                );
+                await expect(createWorker()).rejects.toMatchObject({
+                    statusCode: 403,
+                    message: 'Worker limit reached (max 100)',
+                    legacyCode: 'forbidden',
+                });
+            } finally {
+                driverConfig.events = { userHandlers: true };
+            }
+        });
+
+        it('refuses a free user one unit past the cap, across all three sources', async () => {
+            for (const held of [
+                { workers: FREE_CAP },
+                { scopes: FREE_CAP },
+                { owned: FREE_CAP },
+                { workers: 10, scopes: 4, owned: 2 },
+            ]) {
+                holding(DEFAULT_FREE_SUBSCRIPTION, held);
+                await expect(createWorker()).rejects.toMatchObject({
+                    statusCode: 403,
+                    legacyCode: 'forbidden',
+                });
+            }
+        });
+
+        it('lets a free user create up to the cap', async () => {
+            holding(DEFAULT_FREE_SUBSCRIPTION, {
+                workers: 10,
+                scopes: 3,
+                owned: 2,
+            });
+            await expect(createWorker()).rejects.not.toMatchObject({
+                statusCode: 403,
+            });
+        });
+
+        it('refuses a paid user the unit after 100', async () => {
+            holding('paid-plan', { workers: 90, scopes: 5, owned: 5 });
+            await expect(createWorker()).rejects.toMatchObject({
+                statusCode: 403,
+            });
+            holding('paid-plan', { workers: 90, scopes: 5, owned: 4 });
+            await expect(createWorker()).rejects.not.toMatchObject({
+                statusCode: 403,
+            });
+            expect(PAID_CAP).toBe(100);
+        });
+
+        it('keeps a free user over the cap, redeploys what they hold, and refuses a new one', async () => {
+            holding(DEFAULT_FREE_SUBSCRIPTION, { workers: 20 });
+            const user = actor.user;
+            const exists = vi
+                .spyOn(server.stores.subdomain, 'getBySubdomain')
+                .mockImplementation(
+                    async (name: string) =>
+                        (name === 'workers.puter.capped'
+                            ? { user_id: user.id, uuid: 'existing-uuid' }
+                            : null) as never,
+                );
+            const noApp = makeActor({ app: undefined });
+            const create = (workerName: string) =>
+                inCtx(
+                    () =>
+                        target.create({
+                            appId: '',
+                            workerName,
+                            filePath: '/test.js',
+                        }),
+                    noApp,
+                );
+
+            await expect(create('capped')).rejects.not.toMatchObject({
+                statusCode: 403,
+            });
+            await expect(create('another')).rejects.toMatchObject({
+                statusCode: 403,
+            });
+            exists.mockRestore();
+        });
+    });
+
     describe('admission bypass', () => {
         const unverified = (): Actor =>
             makeActor({

src/backend/drivers/workers/WorkerDriver.ts

@@ -43,6 +43,7 @@ import { PuterDriver } from '../types.js';
 import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
 import { awaitLater } from '../../util/concurrency.js';
 import { isUniqueViolation } from '../../util/dbError.js';
+import { assertRentedUnitRoom } from '../../util/rentedUnits.js';
 import { loadFileInput } from '../util/fileInput.js';
 import {
     normalizeLimit,
@@ -57,6 +58,7 @@ const CF_BASE_URL = 'https://api.cloudflare.com/client/v4/accounts';
 export const WORKER_COMPATIBILITY_DATE = '2025-07-15';
 export const WORKER_COMPATIBILITY_FLAGS = ['global_fetch_strictly_public'];
 const WORKER_NAME_REGEX = /^[a-zA-Z0-9_-]+$/;
+/** Cap on ordinary workers while user-scope handlers are off. */
 const MAX_WORKERS_PER_USER = 100;
 
 /**
@@ -179,12 +181,17 @@ export interface InternalDeployTarget {
      * scripts belong to what.
      */
     tags?: string[];
+    /**
+     * Per-invocation CPU limit in milliseconds, sent with the upload. Unset
+     * sends no limit. Ignored by the local runtime.
+     */
+    cpuMs?: number;
 }
 
 const MAX_SOURCE_SIZE = 10 * 1024 * 1024; // 10 MB
 // How far to scan an app's child apps when resolving which workers it may
-// see. A user is capped at MAX_WORKERS_PER_USER workers, so only that many
-// child apps can actually own one; the ceiling is generous slack over that.
+// see. A user is capped at their plan's rented units, so only that many
+// child apps can actually own a worker; the ceiling is generous slack over that.
 const CHILD_APP_SCAN_LIMIT = 1000;
 
 // -- Preamble --------------------------------------------------------
@@ -452,26 +459,34 @@ export class WorkerDriver extends PuterDriver {
 
         this.#requireCfConfig();
 
-        // Quota check — count existing workers.puter.* subdomains owned by user
-        //
-        // A deploy target creates no such row, so this count can neither see
-        // its scripts nor be grown by them; running it would only make an
-        // unrelated worker quota decide whether these deploy. Whatever bounds
-        // them has to be counted where they actually live, and belongs to the
-        // caller that owns their lifecycle.
-        const existingWorkers =
-            skipAdmission || deployTarget
-                ? []
-                : await this.stores.subdomain.listByUserIdAndPrefix(
-                      actor.user.id,
-                      WORKER_SUBDOMAIN_PREFIX,
-                  );
-        if (existingWorkers.length >= MAX_WORKERS_PER_USER) {
-            throw new HttpError(
-                403,
-                `Worker limit reached (max ${MAX_WORKERS_PER_USER})`,
-                { legacyCode: 'forbidden' },
-            );
+        // With user scopes on, a new worker is one more rented unit under the
+        // plan's cap, and redeploying an existing worker is never refused.
+        // Deploy targets create no worker row and are bounded by their caller.
+        if (this.config.events?.userHandlers === true) {
+            if (!skipAdmission && !deployTarget && !existingSub) {
+                await assertRentedUnitRoom(
+                    {
+                        stores: this.stores,
+                        metering: this.services.metering,
+                    },
+                    actor,
+                );
+            }
+        } else {
+            const existingWorkers =
+                skipAdmission || deployTarget
+                    ? []
+                    : await this.stores.subdomain.listByUserIdAndPrefix(
+                          actor.user.id,
+                          WORKER_SUBDOMAIN_PREFIX,
+                      );
+            if (existingWorkers.length >= MAX_WORKERS_PER_USER) {
+                throw new HttpError(
+                    403,
+                    `Worker limit reached (max ${MAX_WORKERS_PER_USER})`,
+                    { legacyCode: 'forbidden' },
+                );
+            }
         }
 
         // Read source file. loadFileInput runs the read-ACL check internally
@@ -940,6 +955,7 @@ export class WorkerDriver extends PuterDriver {
             compatibility_flags: WORKER_COMPATIBILITY_FLAGS,
             compatibility_date: WORKER_COMPATIBILITY_DATE,
             ...(target?.tags ? { tags: target.tags } : {}),
+            ...(target?.cpuMs ? { limits: { cpu_ms: target.cpuMs } } : {}),
             bindings: [
                 // A script deployed without a token gets no binding at all,
                 // rather than an empty one for its code to find.

src/backend/server.test.ts

@@ -36,6 +36,10 @@ import { Controller, Get } from './core/http/decorators.ts';
 import { HttpError } from './core/http/HttpError.ts';
 import { extension, extensionStore } from './extensions.ts';
 import { PuterServer } from './server.ts';
+import {
+    JobScheduler,
+    type JobDefinition,
+} from './services/jobs/JobScheduler.ts';
 import {
     allocateEphemeralPort,
     createTestUser,
@@ -472,59 +476,6 @@ describe('PuterServer host header validation — permissive modes', () => {
     });
 });
 
-describe('PuterServer behind a trusted proxy', () => {
-    let server: PuterServer;
-    let port: number;
-
-    beforeAll(async () => {
-        port = await allocateEphemeralPort();
-        server = await setupTestServer(
-            {
-                port,
-                domain: 'puter.localhost',
-                origin: `http://puter.localhost:${port}`,
-                api_base_url: `http://api.puter.localhost:${port}`,
-                allow_all_host_values: false,
-                allow_no_host_header: false,
-                custom_domains_enabled: false,
-                static_hosting_domain: 'site.puter.localhost',
-                trust_proxy: 1,
-            } as unknown as IConfig,
-            { listen: true },
-        );
-    });
-
-    afterAll(async () => {
-        await server?.shutdown();
-    });
-
-    it('serves the main domain whatever X-Forwarded-Host names', async () => {
-        const res = await rawRequest(port, '/healthcheck', {
-            host: `puter.localhost:${port}`,
-            'x-forwarded-host': 'nosuchsite.site.puter.localhost',
-        });
-        expect(res.status).toBe(200);
-        expect(res.headers['x-frame-options']).toBe('SAMEORIGIN');
-    });
-
-    it('keeps a hosted-site host on site hosting', async () => {
-        const res = await rawRequest(port, '/healthcheck', {
-            host: `nosuchsite.site.puter.localhost:${port}`,
-            'x-forwarded-host': 'api.puter.localhost',
-        });
-        expect(res.status).toBe(404);
-    });
-
-    it('reads subdomains from Host', async () => {
-        const res = await rawRequest(port, '/healthcheck', {
-            host: `api.puter.localhost:${port}`,
-            'x-forwarded-host': 'puter.localhost',
-        });
-        expect(res.status).toBe(200);
-        expect(res.headers['x-frame-options']).toBeUndefined();
-    });
-});
-
 describe('PuterServer host header validation — custom domains with every host accepted', () => {
     let server: PuterServer;
     let port: number;
@@ -1592,6 +1543,69 @@ describe('PuterServer lifecycle hooks', () => {
     });
 });
 
+describe('PuterServer shutdown during prepare', () => {
+    it('waits for prepare hooks the drain already started before tearing down', async () => {
+        const port = await allocateEphemeralPort();
+        const server = await setupTestServer({ port } as unknown as IConfig, {
+            listen: true,
+        });
+        const order: string[] = [];
+        (server.services as unknown as Record<string, unknown>).slowPrepare = {
+            onServerPrepareShutdown: async () => {
+                order.push('prepare:start');
+                await new Promise((resolve) => setTimeout(resolve, 200));
+                order.push('prepare:done');
+            },
+            onServerShutdown: () => {
+                order.push('shutdown');
+            },
+        };
+
+        await server.prepareShutdown({ drainMs: 0 });
+        // The listener's close callback starts the prepare hooks.
+        await new Promise((resolve) => setTimeout(resolve, 50));
+        await server.shutdown();
+
+        expect(order).toEqual(['prepare:start', 'prepare:done', 'shutdown']);
+    });
+});
+
+/**
+ * An extension may take over `services.jobs` (a fleet-wide scheduler). Core
+ * services register their jobs on it at start, so it must keep the built-in's
+ * first place in the layer.
+ */
+describe('PuterServer jobs replacement', () => {
+    it('lets an extension service registered as `jobs` replace the built-in and keep its first position', async () => {
+        const registered: string[] = [];
+        class ReplacementJobs extends JobScheduler {
+            override register(name: string, definition: JobDefinition) {
+                registered.push(name);
+                super.register(name, definition);
+            }
+        }
+        const started = vi.spyOn(JobScheduler.prototype, 'onServerStart');
+        extension.registerService('jobs', ReplacementJobs as never);
+        let server: PuterServer | undefined;
+        try {
+            server = await setupTestServer();
+            const { jobs } = server.services;
+            expect(jobs).toBeInstanceOf(ReplacementJobs);
+            expect(Object.keys(server.services)[0]).toBe('jobs');
+            expect(extension.import('service').jobs).toBeInstanceOf(
+                ReplacementJobs,
+            );
+            // Only the replacement gets lifecycle hooks.
+            expect(started.mock.contexts).toEqual([jobs]);
+            expect(registered).toContain('auth.sessions.purge');
+        } finally {
+            delete (extensionStore.services as Record<string, unknown>).jobs;
+            started.mockRestore();
+            await server?.shutdown();
+        }
+    });
+});
+
 /**
  * Each layer's `onServerShutdown` has to run while the layers beneath it (the
  * ones it writes through) are still up, so teardown goes top-down: drivers,

src/backend/server.ts

@@ -413,17 +413,6 @@ export class PuterServer {
      *   `#materializeRoute` as those options ship.
      */
     #installGlobalMiddleware() {
-        // -- Forwarded host ------------------------------------------
-        // With `trust proxy` set, express derives `req.hostname` and
-        // `req.subdomains` from X-Forwarded-Host, which any client can send.
-        // Dropping it keeps them on Host, the header host validation checks,
-        // so a request can't pick a different site or subdomain router than
-        // the host it was sent to.
-        this.#app.use((req, _res, next) => {
-            delete req.headers['x-forwarded-host'];
-            next();
-        });
-
         // -- Egress metering -----------------------------------------
         // First, so the byte counter wraps `res.write` before compression
         // does and therefore counts what actually goes out rather than what
@@ -1776,16 +1765,19 @@ export class PuterServer {
         }
     }
 
-    #prepareShutdownHooksRan = false;
+    #prepareShutdownHooks: Promise<void> | null = null;
 
     /**
      * Run every layer's `onServerPrepareShutdown` exactly once, whichever of
-     * `prepareShutdown()` / `shutdown()` gets there first.
+     * `prepareShutdown()` / `shutdown()` gets there first; the other waits for
+     * that run to finish.
      */
-    async #runPrepareShutdownHooks() {
-        if (this.#prepareShutdownHooksRan) return;
-        this.#prepareShutdownHooksRan = true;
-        await this.#runHook('bottomUp', 'onServerPrepareShutdown');
+    #runPrepareShutdownHooks(): Promise<void> {
+        this.#prepareShutdownHooks ??= this.#runHook(
+            'bottomUp',
+            'onServerPrepareShutdown',
+        );
+        return this.#prepareShutdownHooks;
     }
 
     async prepareShutdown(info: ServerDrainInfo = {}) {

src/backend/services/auth/AuthService.test.ts

@@ -24,6 +24,7 @@ import { makeActor, type Actor } from '../../core/actor.js';
 import { PuterServer } from '../../server.js';
 import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js';
 import { setupTestServer } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import { generateDefaultFsentries } from '../../util/userProvisioning.js';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS } from '../permission/consts.js';
@@ -4048,7 +4049,6 @@ describe('AuthService (integration)', () => {
     });
 
     describe('purgeEndedSessions', () => {
-        const GATE_KEY = 'sessions:purge:gate';
         const DAY = 24 * 60 * 60;
 
         const makeSession = async (
@@ -4077,8 +4077,15 @@ describe('AuthService (integration)', () => {
             return rows.length > 0;
         };
 
+        const idOf = async (uuid: string) => {
+            const [row] = await server.clients.db.read(
+                'SELECT `id` FROM `sessions` WHERE `uuid` = ?',
+                [uuid],
+            );
+            return Number(row.id);
+        };
+
         it('deletes rows ended past their retention and keeps the rest', async () => {
-            await server.clients.redis.del(GATE_KEY);
             const user = await makeUser();
             const id = user.id;
             const oldRevoked = await makeSession(id, 'asset', 'revoked_at', 31);
@@ -4100,7 +4107,7 @@ describe('AuthService (integration)', () => {
             const live = await makeSession(id, 'web');
 
             expect(
-                await authService.purgeEndedSessions(),
+                await authService.purgeEndedSessions(testJobContext()),
             ).toBeGreaterThanOrEqual(3);
 
             expect(await exists(oldRevoked)).toBe(false);
@@ -4112,34 +4119,84 @@ describe('AuthService (integration)', () => {
             expect(await exists(live)).toBe(true);
         });
 
-        it('skips while another run holds the gate', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            await server.clients.redis.set(GATE_KEY, 'other-node');
+        it('starts from the saved cursor and clears it once the walk is done', async () => {
             const user = await makeUser();
-            const oldRevoked = await makeSession(
-                user.id,
-                'asset',
-                'revoked_at',
-                31,
-            );
+            const below = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const above = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const ctx = testJobContext({ afterId: await idOf(below) });
 
-            expect(await authService.purgeEndedSessions()).toBeNull();
-            expect(await exists(oldRevoked)).toBe(true);
-            await server.clients.redis.del(GATE_KEY);
+            expect(
+                await authService.purgeEndedSessions(ctx),
+            ).toBeGreaterThanOrEqual(1);
+            expect(await exists(below)).toBe(true);
+            expect(await exists(above)).toBe(false);
+            expect(ctx.saved).toEqual([null]);
         });
 
-        it('frees the gate when the walk fails', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            const spy = vi
-                .spyOn(server.clients.db, 'read')
-                .mockRejectedValueOnce(new Error('db down'));
+        it('clears its own checkpoints once a walk completes, so the next run starts at zero', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 11,
+            );
+            const ranges: number[] = [];
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    return 0;
+                });
             try {
-                await expect(authService.purgeEndedSessions()).rejects.toThrow(
-                    'db down',
+                const first = testJobContext();
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(11);
+                // Checkpointed at ten ranges, then cleared at the end.
+                expect(first.saved).toEqual([{ afterId: SPAN * 10 }, null]);
+
+                ranges.length = 0;
+                await authService.purgeEndedSessions(
+                    testJobContext(first.saved.at(-1)),
                 );
-                expect(await server.clients.redis.get(GATE_KEY)).toBeNull();
+                expect(ranges[0]).toBe(0);
+                expect(ranges).toHaveLength(11);
             } finally {
-                spy.mockRestore();
+                purge.mockRestore();
+                vi.restoreAllMocks();
+            }
+        });
+
+        it('stops between ranges once aborted, and the next run resumes at the saved cursor', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 30,
+            );
+            const ranges: number[] = [];
+            const first = testJobContext();
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    // Aborted mid-walk, as a lost lease or a shutdown would.
+                    if (ranges.length === 12) first.abort();
+                    return 0;
+                });
+            try {
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(12);
+                // Saved every 10 ranges, and where it stopped.
+                expect(first.saved).toEqual([
+                    { afterId: SPAN * 10 },
+                    { afterId: SPAN * 12 },
+                ]);
+
+                ranges.length = 0;
+                const next = testJobContext(first.saved.at(-1));
+                await authService.purgeEndedSessions(next);
+                expect(ranges[0]).toBe(SPAN * 12);
+                expect(ranges.at(-1)).toBe(SPAN * 29);
+                expect(next.saved.at(-1)).toBeNull();
+            } finally {
+                purge.mockRestore();
+                vi.restoreAllMocks();
             }
         });
     });

src/backend/services/auth/AuthService.ts

@@ -46,6 +46,7 @@ import type { puterServices } from '../index';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS, PERMISSION_MAX_LEN } from '../permission/consts';
 import type { GrantMeta } from '../permission/PermissionService.js';
+import { savedAfterId, type JobRunContext } from '../jobs/JobScheduler.js';
 import { PuterService } from '../types';
 import type {
     AccessTokenPayload,
@@ -64,13 +65,14 @@ const MAX_ORIGIN_UID_GENERATIONS = 8;
 // rows double as sign-in history, so they stay longer.
 const SESSION_RETENTION_SECONDS = 30 * 24 * 60 * 60;
 const WEB_SESSION_RETENTION_SECONDS = 365 * 24 * 60 * 60;
-// Every node checks hourly; the gate lets one node through per week.
-const SESSION_PURGE_CHECK_MS = 60 * 60 * 1000;
-const SESSION_PURGE_GATE_KEY = 'sessions:purge:gate';
-const SESSION_PURGE_GATE_SECONDS = 7 * 24 * 60 * 60;
+const SESSION_PURGE_JOB = 'auth.sessions.purge';
+const SESSION_PURGE_EVERY_MS = 7 * 24 * 60 * 60 * 1000;
+const SESSION_PURGE_LEASE_MS = 30 * 60 * 1000;
 // Ids per DELETE, with a pause between, so replicas keep up.
 const SESSION_PURGE_SPAN = 5000;
 const SESSION_PURGE_PAUSE_MS = 50;
+/** Ranges walked between saves of the cursor. */
+const SESSION_PURGE_SAVE_EVERY = 10;
 
 const nowSeconds = (): number => Math.floor(Date.now() / 1000);
 
@@ -138,11 +140,17 @@ export interface AuthResult {
 export class AuthService extends PuterService {
     declare protected services: LayerInstances<typeof puterServices>;
 
-    #sessionPurge: ReturnType<typeof setInterval> | null = null;
-    #stopping = false;
-
     override onServerStart(): void {
-        this.#armSessionPurge();
+        this.services.jobs.register(SESSION_PURGE_JOB, {
+            every: SESSION_PURGE_EVERY_MS,
+            scope: 'fleet',
+            nearPrimary: true,
+            leaseMs: SESSION_PURGE_LEASE_MS,
+            run: async (ctx) => {
+                const purged = await this.purgeEndedSessions(ctx);
+                console.log(`[auth] purged ${purged} ended sessions`);
+            },
+        });
 
         // Users implicitly hold read access to their own email — needed for
         // any permission-gated path that asks for `user:<uuid>:email:read`
@@ -164,85 +172,54 @@ export class AuthService extends PuterService {
         });
     }
 
-    override onServerPrepareShutdown(): void {
-        this.#stopping = true;
-        if (this.#sessionPurge) clearInterval(this.#sessionPurge);
-        this.#sessionPurge = null;
-    }
-
     // -- Session purge -----------------------------------------------
 
     /**
      * Delete sessions revoked or expired longer ago than their retention,
-     * walking the table by id. Returns the rows deleted, or null when another
-     * node already holds this week's gate.
+     * walking the table by id from the cursor an interrupted run saved. Returns
+     * the rows deleted; stops early once `signal` aborts.
      */
-    async purgeEndedSessions(): Promise<number | null> {
-        const claimed = await this.clients.redis.set(
-            SESSION_PURGE_GATE_KEY,
-            String(Date.now()),
-            'EX',
-            SESSION_PURGE_GATE_SECONDS,
-            'NX',
-        );
-        if (claimed !== 'OK') return null;
-
+    async purgeEndedSessions({
+        signal,
+        state,
+        saveState,
+    }: JobRunContext): Promise<number> {
         const now = nowSeconds();
         const cutoffs = {
             cutoff: now - SESSION_RETENTION_SECONDS,
             webCutoff: now - WEB_SESSION_RETENTION_SECONDS,
         };
+        const maxId = await this.stores.session.maxId();
+        let afterId = savedAfterId(state);
         let purged = 0;
-        let finished = false;
-        try {
-            const maxId = await this.stores.session.maxId();
-            for (
-                let afterId = 0;
-                afterId < maxId;
-                afterId += SESSION_PURGE_SPAN
-            ) {
-                if (this.#stopping) return purged;
-                purged += await this.stores.session.purgeEndedInRange(
-                    afterId,
-                    SESSION_PURGE_SPAN,
-                    cutoffs,
-                );
-                await new Promise((resolve) =>
-                    setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
-                );
-            }
-            finished = true;
-            return purged;
-        } finally {
-            // A walk cut short frees the gate, so another node restarts it on
-            // its next check instead of next week.
-            if (!finished) {
+        let unsaved = 0;
+        while (afterId < maxId) {
+            if (signal.aborted) {
+                // Kept for the next run if this one still holds the job.
                 try {
-                    await this.clients.redis.del(SESSION_PURGE_GATE_KEY);
+                    await saveState({ afterId });
                 } catch {
-                    // The gate expires on its own.
+                    // Lost the job; the next run redoes these ranges.
                 }
+                return purged;
             }
-        }
-    }
-
-    #armSessionPurge(): void {
-        const timer = setInterval(() => {
-            void this.#runSessionPurge();
-        }, SESSION_PURGE_CHECK_MS);
-        timer.unref?.();
-        this.#sessionPurge = timer;
-    }
-
-    async #runSessionPurge(): Promise<void> {
-        try {
-            const purged = await this.purgeEndedSessions();
-            if (purged !== null) {
-                console.log(`[auth] purged ${purged} ended sessions`);
+            purged += await this.stores.session.purgeEndedInRange(
+                afterId,
+                SESSION_PURGE_SPAN,
+                cutoffs,
+            );
+            afterId += SESSION_PURGE_SPAN;
+            if (++unsaved >= SESSION_PURGE_SAVE_EVERY) {
+                await saveState({ afterId });
+                unsaved = 0;
             }
-        } catch (err) {
-            console.warn('[auth] session purge failed', err);
+            await new Promise((resolve) =>
+                setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
+            );
         }
+        // Clears any checkpoint, this run's included, so next week starts at 0.
+        await saveState(null);
+        return purged;
     }
 
     // -- Public API --------------------------------------------------
@@ -2887,7 +2864,7 @@ export class AuthService extends PuterService {
 
     #actorUserFromRow(user: UserRow) {
         // Strip the password hash; pass everything else through so callers
-        // can read metadata, desktop_bg_*, otp_enabled, etc. without
+        // can read metadata, otp_enabled, etc. without
         // re-fetching the user. Mirrors what /whoami exposes off of UserRow.
         const { password: _password, ...rest } = user;
         return {

src/backend/services/events/EventsService.test.ts

@@ -185,7 +185,11 @@ const aclService = () => ({
         !denied.has(resource.path) && !deniedHolders.has(actor.user?.id ?? -1),
     getSafeAclError: async (_actor: Actor, resource: { path: string }) =>
         denied.get(resource.path) === 'forbidden'
-            ? { status: 403, message: 'Forbidden', fields: { code: 'forbidden' } }
+            ? {
+                  status: 403,
+                  message: 'Forbidden',
+                  fields: { code: 'forbidden' },
+              }
             : {
                   status: 404,
                   message: 'Subject does not exist',
@@ -281,8 +285,8 @@ interface ForwardedCall {
 }
 
 /**
- * Each service gets its own outbox. A delivery still in flight when a test
- * ends must land in that test's record, not in the next one's.
+ * Each service gets its own outbox. A delivery still in flight when a test ends
+ * must land in that test's record, not in the next one's.
  */
 const buildService = (
     config: IConfig,
@@ -293,12 +297,14 @@ const buildService = (
     metered: MeteredLine[];
     forwarded: ForwardedCall[];
     eventBus: { on: ReturnType<typeof vi.fn>; emit: ReturnType<typeof vi.fn> };
+    jobs: { register: ReturnType<typeof vi.fn> };
 } => {
     const outbox: Array<{ socket?: string; envelope: DeliveryEnvelope }> = [];
     const counted: DeliveryEnvelope[] = [];
     const lines: MeteredLine[] = [];
     const forwardedCalls: ForwardedCall[] = [];
     const bus = { on: vi.fn(), emit: vi.fn() };
+    const jobs = { register: vi.fn() };
     const built = new EventsService(
         config,
         {
@@ -315,6 +321,7 @@ const buildService = (
             kvShareHandle: kvShareHandleStore,
         } as never,
         {
+            jobs,
             eventForward: {
                 // A deployment with no peers has nowhere to forward to, which
                 // is what every test here is.
@@ -347,10 +354,7 @@ const buildService = (
             acl: aclService(),
             permission: permissionService(),
             metering: {
-                bufferIncrementUsages: (
-                    actor: Actor,
-                    usages: UsageInput[],
-                ) => {
+                bufferIncrementUsages: (actor: Actor, usages: UsageInput[]) => {
                     for (const usage of usages)
                         lines.push({
                             userUuid: actor.user?.uuid,
@@ -376,6 +380,7 @@ const buildService = (
         metered: lines,
         forwarded: forwardedCalls,
         eventBus: bus,
+        jobs,
     };
 };
 
@@ -564,6 +569,59 @@ describe('the feature switch', () => {
         );
     });
 
+    it('hands its table sweeps to the job scheduler and arms only the pending sweep itself', () => {
+        const intervals = vi.spyOn(globalThis, 'setInterval');
+        const timeouts = vi.spyOn(globalThis, 'setTimeout');
+        let jobs: { register: ReturnType<typeof vi.fn> };
+        try {
+            ({ jobs } = buildService({ events: { enabled: true } } as IConfig));
+            // The pending sweep is the region's queue, so every node runs it.
+            expect(intervals.mock.calls.map(([, ms]) => ms)).toEqual([10_000]);
+            expect(timeouts).not.toHaveBeenCalled();
+        } finally {
+            intervals.mockRestore();
+            timeouts.mockRestore();
+        }
+        expect(
+            jobs.register.mock.calls.map(([name, job]) => ({
+                name,
+                every: job.every,
+                scope: job.scope,
+                nearPrimary: job.nearPrimary ?? false,
+                leaseMs: job.leaseMs,
+            })),
+        ).toEqual([
+            {
+                name: 'events.subscriptions.expirySweep',
+                every: 60 * 60_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: 15 * 60_000,
+            },
+            {
+                name: 'events.userHandlers.gc',
+                every: 60 * 60_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: 15 * 60_000,
+            },
+            {
+                name: 'events.subscriptions.creditSweep',
+                every: 15 * 60_000,
+                scope: 'fleet',
+                nearPrimary: false,
+                leaseMs: 10 * 60_000,
+            },
+        ]);
+    });
+
+    it('registers no job when off', () => {
+        const { jobs } = buildService({
+            events: { enabled: false },
+        } as IConfig);
+        expect(jobs.register).not.toHaveBeenCalled();
+    });
+
     it('spends nothing at all on a dispatch when off', async () => {
         const { service: off } = buildService({
             events: { enabled: false },
@@ -1206,7 +1264,9 @@ describe('matching', () => {
     it('stops delivering to a durable row the moment it expires', async () => {
         const { documents, file } = seedTree();
         const now = Math.floor(Date.now() / 1000);
-        const row = (over: Partial<DurableSubscription>): DurableSubscription => ({
+        const row = (
+            over: Partial<DurableSubscription>,
+        ): DurableSubscription => ({
             subId: `durable-${seq}-${over.expiresAt}`,
             token: `f#${documents.uid}`,
             ownerUserId: userId,
@@ -1355,9 +1415,9 @@ describe('matching', () => {
         await dispatch(closed);
         await flush();
 
-        expect(sent.map((s) => (s.envelope.event as { uid: string }).uid)).toEqual([
-            reachable.uid,
-        ]);
+        expect(
+            sent.map((s) => (s.envelope.event as { uid: string }).uid),
+        ).toEqual([reachable.uid]);
     });
 
     it('publishes nothing for an event with no registry entry', async () => {
@@ -1399,9 +1459,9 @@ describe('matching', () => {
         await flush();
 
         expect(sent).toHaveLength(1);
-        expect(
-            (sent[0].envelope.event as { path: string }).path,
-        ).toBe(inside.path);
+        expect((sent[0].envelope.event as { path: string }).path).toBe(
+            inside.path,
+        );
     });
 
     it('does not reach a sibling that only shares the missing name as a prefix', async () => {
@@ -1473,8 +1533,9 @@ describe('coalescing', () => {
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
         expect(sent).toHaveLength(2);
-        expect(sent.map((s) => (s.envelope.event as { uid: string }).uid).sort())
-            .toEqual([file.uid, second.uid].sort());
+        expect(
+            sent.map((s) => (s.envelope.event as { uid: string }).uid).sort(),
+        ).toEqual([file.uid, second.uid].sort());
     });
 
     it('counts exactly what it delivered', async () => {
@@ -1620,9 +1681,9 @@ describe('limits', () => {
             }
         }
 
-        await expect(subscribe(`fs:${documents.uid}`)).resolves.toMatchObject(
-            { subId: expect.any(String) },
-        );
+        await expect(subscribe(`fs:${documents.uid}`)).resolves.toMatchObject({
+            subId: expect.any(String),
+        });
     });
 
     it('refuses unsubscribes past their own per-minute budget', async () => {
@@ -1691,9 +1752,7 @@ describe('limits', () => {
             ),
         );
 
-        expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(
-            1,
-        );
+        expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
         expect(results.filter((r) => r.status === 'rejected')).toHaveLength(
             attempts - 1,
         );
@@ -1890,9 +1949,9 @@ describe('limits', () => {
         ).toHaveLength(EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT);
         // 200 evaluated, 20 never reached: everything past the delivery cap is
         // told it missed something, itself capped.
-        expect(
-            sent.filter((s) => s.envelope.event.op === 'gap'),
-        ).toHaveLength(EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT);
+        expect(sent.filter((s) => s.envelope.event.op === 'gap')).toHaveLength(
+            EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT,
+        );
         expect(
             sent.find((s) => s.envelope.event.op === 'gap')?.envelope.event,
         ).toMatchObject({ reason: 'filter_evaluation_limit' });
@@ -1943,8 +2002,7 @@ describe('the socket surface', () => {
             fire: (event, ...args) =>
                 (
                     handlers.get(event) as
-                        | ((...a: unknown[]) => void)
-                        | undefined
+                        ((...a: unknown[]) => void) | undefined
                 )?.(...args),
         };
     };
@@ -2125,10 +2183,14 @@ it('names the delivery channel the clients listen on', () => {
 
 describe('asking a kv subscription for the value', () => {
     it('is recorded on the row and reported in the view', async () => {
-        const { sub } = await service.subscribe(appActorFor(OWN_APP), socketId, {
-            subject: `kv:${OWN_APP}:cart`,
-            includeValue: true,
-        });
+        const { sub } = await service.subscribe(
+            appActorFor(OWN_APP),
+            socketId,
+            {
+                subject: `kv:${OWN_APP}:cart`,
+                includeValue: true,
+            },
+        );
         expect(sub.includeValue).toBe(true);
 
         const plain = await subscribeKv(`kv:${OWN_APP}:cart`);
@@ -2211,10 +2273,7 @@ describe('resolving a kv subject', () => {
     });
 
     it('refuses a pattern the grammar cannot enumerate from a key', async () => {
-        for (const subject of [
-            `kv:${OWN_APP}:ca*rt`,
-            `kv:${OWN_APP}:cart?`,
-        ])
+        for (const subject of [`kv:${OWN_APP}:ca*rt`, `kv:${OWN_APP}:cart?`])
             await expect(subscribeKv(subject)).rejects.toSatisfy(
                 (err: unknown) =>
                     isHttpError(err) && err.legacyCode === 'invalid_kv_pattern',
@@ -2329,8 +2388,7 @@ describe('delivering a kv change', () => {
 
     const eventFor = (subId: string) =>
         sent.find((one) => one.envelope.subId === subId)?.envelope.event as
-            | Record<string, unknown>
-            | undefined;
+            Record<string, unknown> | undefined;
 
     it('hands the value to the row that asked for it and to no other', async () => {
         vi.useFakeTimers();
@@ -2445,7 +2503,9 @@ describe('delivering a kv change', () => {
         await dispatchKv(['cart'], { values: ['kept'] });
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
-        const deliveries = sent.filter((one) => one.envelope.event.op === 'set');
+        const deliveries = sent.filter(
+            (one) => one.envelope.event.op === 'set',
+        );
         const gaps = sent.filter((one) => one.envelope.event.op === 'gap');
         expect(deliveries).toHaveLength(128);
         expect(gaps).toHaveLength(1);
@@ -2492,12 +2552,12 @@ describe('delivering a kv change', () => {
         await dispatchKv(['cart']);
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
-        expect(sent.filter((one) => one.envelope.event.op === 'set')).toHaveLength(
-            EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.limit,
-        );
-        expect(sent.filter((one) => one.envelope.event.op === 'gap')).toHaveLength(
-            1,
-        );
+        expect(
+            sent.filter((one) => one.envelope.event.op === 'set'),
+        ).toHaveLength(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.limit);
+        expect(
+            sent.filter((one) => one.envelope.event.op === 'gap'),
+        ).toHaveLength(1);
     });
 
     it('uses the free KV limit when owner policy lookup fails', async () => {
@@ -2508,7 +2568,9 @@ describe('delivering a kv change', () => {
         await dispatchKv(['cart']);
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
-        expect(sent.filter((one) => one.envelope.event.op === 'set')).toHaveLength(
+        expect(
+            sent.filter((one) => one.envelope.event.op === 'set'),
+        ).toHaveLength(
             EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.bySubscription[
                 DEFAULT_FREE_SUBSCRIPTION
             ],
@@ -2521,7 +2583,9 @@ describe('delivering a kv change', () => {
         await seedKvSubscriptions(
             EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT.bySubscription[
                 DEFAULT_FREE_SUBSCRIPTION
-            ] - 128 + 1,
+            ] -
+                128 +
+                1,
             { includeValue: true, match: 'elsewhere', tag: 'filtered-out' },
         );
 
@@ -2543,7 +2607,9 @@ describe('delivering a kv change', () => {
         });
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
-        const deliveries = sent.filter((one) => one.envelope.event.op === 'set');
+        const deliveries = sent.filter(
+            (one) => one.envelope.event.op === 'set',
+        );
         expect(deliveries).toHaveLength(128);
         expect(
             deliveries.every(
@@ -2769,8 +2835,7 @@ describe('what a kv dispatch costs', () => {
 });
 
 describe('the cross-app kv gate', () => {
-    const grantRead = (app: string) =>
-        grants.add(`app-data:${app}:kv:read`);
+    const grantRead = (app: string) => grants.add(`app-data:${app}:kv:read`);
 
     const crossAppService = () =>
         buildService({
@@ -2834,7 +2899,8 @@ describe('the cross-app kv gate', () => {
         ({ service } = crossAppService());
 
         await expect(subscribeKv(`kv:${OTHER_APP}:cart`)).rejects.toSatisfy(
-            (err: unknown) => isHttpError(err) && err.legacyCode === 'forbidden',
+            (err: unknown) =>
+                isHttpError(err) && err.legacyCode === 'forbidden',
         );
     });
 
@@ -2845,8 +2911,7 @@ describe('the cross-app kv gate', () => {
 
         await expect(subscribeKv('kv:app-gone:cart')).rejects.toSatisfy(
             (err: unknown) =>
-                isHttpError(err) &&
-                err.legacyCode === 'subject_does_not_exist',
+                isHttpError(err) && err.legacyCode === 'subject_does_not_exist',
         );
     });
 
@@ -2860,7 +2925,8 @@ describe('the cross-app kv gate', () => {
         });
 
         await expect(subscribeKv(`kv:${OTHER_APP}:cart`)).rejects.toSatisfy(
-            (err: unknown) => isHttpError(err) && err.legacyCode === 'forbidden',
+            (err: unknown) =>
+                isHttpError(err) && err.legacyCode === 'forbidden',
         );
     });
 
@@ -3135,9 +3201,7 @@ describe('cross-user kv handles', () => {
         expect(off.kvHandlesEnabled).toBe(false);
         expect(service.kvHandlesEnabled).toBe(true);
 
-        await expect(
-            subscribeAsGuest(`kv:${handle}:*`, off),
-        ).rejects.toSatisfy(
+        await expect(subscribeAsGuest(`kv:${handle}:*`, off)).rejects.toSatisfy(
             (err: unknown) =>
                 isHttpError(err) &&
                 err.legacyCode === 'events_kv_handles_disabled',
@@ -3345,10 +3409,13 @@ describe('cross-user kv handles', () => {
     it.each([
         ['an unknown handle', () => undefined],
         ['a revoked handle', () => mintHandle({ revokedAt: 1 })],
-        ['a handle whose grant the caller does not hold', () => {
-            mintHandle();
-            grants.delete(permission);
-        }],
+        [
+            'a handle whose grant the caller does not hold',
+            () => {
+                mintHandle();
+                grants.delete(permission);
+            },
+        ],
     ])('answers %s as absent', async (_case, setUp) => {
         setUp();
         await expect(subscribeAsGuest(`kv:${handle}:*`)).rejects.toSatisfy(

src/backend/services/events/appDeletion.integration.test.ts

@@ -19,19 +19,29 @@
 
 /**
  * What deleting an app leaves behind on the events side: its durable rows,
- * their backlog and its published handlers can never deliver again — the
- * grant identity they are re-checked under no longer resolves — so
- * `settleDeletedApp` tears them down on the `app.changed` bus key rather than
- * leaving them to cost their holder an anchor slot forever. Also the hourly
- * sweep that catches whatever neither this nor `workers.destroy` reached.
+ * their backlog and its published handlers can never deliver again — the grant
+ * identity they are re-checked under no longer resolves — so `settleDeletedApp`
+ * tears them down on the `app.changed` bus key rather than leaving them to cost
+ * their holder an anchor slot forever. Also the hourly sweep that catches
+ * whatever neither this nor `workers.destroy` reached.
  */
 
-import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import {
+    afterAll,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
 import { v4 as uuidv4 } from 'uuid';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import { EVENTS_WORKER_SESSION_NAME } from './workerRuntime.js';
+import { appHandlerScope } from '../../stores/events/EventHandlerStore.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 
@@ -125,10 +135,13 @@ const emitDeleted = async (appUid: string): Promise<void> => {
 const publish = (
     token: string,
     body: { appUid: string; name: string; source: string },
-): Promise<ApiResponse> => call('POST', '/events/handlers/publish', token, body);
+): Promise<ApiResponse> =>
+    call('POST', '/events/handlers/publish', token, body);
 
-/** A durable `single` subscription bound to a published handler, with the
- * grants a background delivery needs. Returns the subscribed subId. */
+/**
+ * A durable `single` subscription bound to a published handler, with the grants
+ * a background delivery needs. Returns the subscribed subId.
+ */
 const subscribeBackgroundHandler = async (
     app: { uid: string; token: string },
     handlerName: string,
@@ -162,6 +175,87 @@ const subscribeBackgroundHandler = async (
     return subscribed.body.subId as string;
 };
 
+describe('user scopes of a deleted app or account', () => {
+    const publishUserHandler = (
+        appUid: string,
+        holder: number,
+        name = 'mine',
+    ) =>
+        env.server.stores.eventHandler.publish({
+            scope: { appUid, holder },
+            name,
+            source: SOURCE,
+            broadcast: false,
+        });
+    const userHandlers = (appUid: string, holder: number) =>
+        env.server.stores.eventHandler.listForApp({ appUid, holder });
+
+    let destroyed: Array<{ appUid: string; scope: string; userId: number }>;
+    beforeAll(() => {
+        env.server.clients.event.on('events.worker.destroy', (_key, event) => {
+            destroyed?.push({
+                appUid: event.appUid,
+                scope: event.scope,
+                userId: event.actor.user.id,
+            });
+        });
+    });
+    beforeEach(async () => {
+        destroyed = [];
+        await env.server.clients.db.write(
+            'DELETE FROM `event_user_handlers`',
+            [],
+        );
+    });
+
+    it('removes every holder`s scope with the app, announcing each as the holder', async () => {
+        const app = await makeApp(userId);
+        await publishUserHandler(app.uid, userId);
+        await publishUserHandler(app.uid, otherUserId);
+        await publishUserHandler(app.uid, otherUserId, 'second');
+        const untouched = await makeApp(userId);
+        await publishUserHandler(untouched.uid, userId);
+
+        await emitDeleted(app.uid);
+
+        expect(await userHandlers(app.uid, userId)).toEqual([]);
+        expect(await userHandlers(app.uid, otherUserId)).toEqual([]);
+        expect(
+            destroyed
+                .filter((event) => event.scope === 'user')
+                .map((event) => [event.appUid, event.userId])
+                .sort(),
+        ).toEqual(
+            [
+                [app.uid, userId],
+                [app.uid, otherUserId],
+            ].sort(),
+        );
+        // Another app's scope is not touched.
+        expect(await userHandlers(untouched.uid, userId)).toHaveLength(1);
+    });
+
+    it('removes a deleted account`s scopes in every app, and no one else`s', async () => {
+        const first = await makeApp(userId);
+        const second = await makeApp(userId);
+        for (const app of [first, second]) {
+            await publishUserHandler(app.uid, otherUserId);
+            await publishUserHandler(app.uid, userId);
+        }
+
+        await env.server.clients.event.emitAndWait(
+            'user.delete',
+            { user_id: otherUserId, appUids: [] },
+            {},
+        );
+
+        for (const app of [first, second]) {
+            expect(await userHandlers(app.uid, otherUserId)).toEqual([]);
+            expect(await userHandlers(app.uid, userId)).toHaveLength(1);
+        }
+    });
+});
+
 describe('an app deleted while it has events state', () => {
     it('clears the rows, the backlog and the handlers when the app is deleted', async () => {
         const app = await makeApp(userId);
@@ -186,15 +280,17 @@ describe('an app deleted while it has events state', () => {
 
         await emitDeleted(app.uid);
 
-        const listed = await env.server.stores.durableSubscription.listForHolder(
-            userId,
-            { appUid: app.uid },
-        );
+        const listed =
+            await env.server.stores.durableSubscription.listForHolder(userId, {
+                appUid: app.uid,
+            });
         expect(listed.items).toEqual([]);
         expect(await env.server.stores.pendingDelivery.depth(subId)).toBe(0);
-        expect(await env.server.stores.eventHandler.listForApp(app.uid)).toEqual(
-            [],
-        );
+        expect(
+            await env.server.stores.eventHandler.listForApp(
+                appHandlerScope(app.uid),
+            ),
+        ).toEqual([]);
     });
 
     it("clears the rows and handlers of a deleted account's apps, and the account's subscription keys", async () => {
@@ -217,14 +313,16 @@ describe('an app deleted while it has events state', () => {
             {},
         );
 
-        const listed = await env.server.stores.durableSubscription.listForHolder(
-            userId,
-            { appUid: app.uid },
-        );
+        const listed =
+            await env.server.stores.durableSubscription.listForHolder(userId, {
+                appUid: app.uid,
+            });
         expect(listed.items).toEqual([]);
-        expect(await env.server.stores.eventHandler.listForApp(app.uid)).toEqual(
-            [],
-        );
+        expect(
+            await env.server.stores.eventHandler.listForApp(
+                appHandlerScope(app.uid),
+            ),
+        ).toEqual([]);
         expect(
             await env.server.stores.eventSubscription.isRegionWarm(goneUserId),
         ).toBe(false);
@@ -259,12 +357,10 @@ describe('an app deleted while it has events state', () => {
             expiresAt: null,
         });
 
-        const { actor: ownerActor } = await env.server.services.auth.authenticate(
-            env.users.user.token,
-        );
-        const { actor: otherActor } = await env.server.services.auth.authenticate(
-            env.users.other.token,
-        );
+        const { actor: ownerActor } =
+            await env.server.services.auth.authenticate(env.users.user.token);
+        const { actor: otherActor } =
+            await env.server.services.auth.authenticate(env.users.other.token);
         const ownerToken = await env.server.services.auth.createWorkerAppToken(
             ownerActor!,
             app.uid,
@@ -327,7 +423,9 @@ describe('an app deleted while it has events state', () => {
         await env.server.stores.app.delete((row as { id: number }).id);
 
         const revoked =
-            await env.server.services.events.sweepStrandedWorkerSessions();
+            await env.server.services.events.sweepStrandedWorkerSessions(
+                testJobContext(),
+            );
         expect(revoked).toBeGreaterThanOrEqual(1);
 
         // `authenticate()` can no longer resolve the token at all — the app
@@ -366,9 +464,60 @@ describe('an app deleted while it has events state', () => {
             EVENTS_WORKER_SESSION_NAME,
         );
 
-        await env.server.services.events.sweepStrandedWorkerSessions();
+        await env.server.services.events.sweepStrandedWorkerSessions(
+            testJobContext(),
+        );
 
         const reauth = await env.server.services.auth.authenticate(token);
         expect(reauth).not.toHaveProperty('reauth');
     });
+
+    it('reaches a stranded session past one run`s two pages on the next run, then wraps', async () => {
+        const db = env.server.clients.db;
+        const now = Math.floor(Date.now() / 1000);
+        const meta = JSON.stringify({
+            worker_name: EVENTS_WORKER_SESSION_NAME,
+        });
+        const workerSession = (appUid: string) => ({
+            statement:
+                'INSERT INTO `sessions` (`uuid`, `user_id`, `meta`, `last_activity`, `created_at`, `kind`, `app_uid`) ' +
+                "VALUES (?, ?, ?, ?, ?, 'worker', ?)",
+            values: [uuidv4(), userId, meta, now, now, appUid],
+        });
+        // A run reads two pages of 500; these fill them, all still standing.
+        const live = Array.from(
+            { length: 1000 },
+            (_, i) => `app-live-${uuidv4().slice(0, 8)}-${i}`,
+        );
+        await db.batchWrite(
+            live.map((uid) => ({
+                statement:
+                    'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
+                values: [uid, uid, uid, `https://${uid}.example/`, userId],
+            })),
+        );
+        await db.batchWrite(live.map(workerSession));
+        const stranded = workerSession(`app-gone-${uuidv4()}`);
+        await db.batchWrite([stranded]);
+        const strandedRevoked = async () => {
+            const [row] = await db.pread(
+                'SELECT `revoked_at` FROM `sessions` WHERE `uuid` = ?',
+                [stranded.values[0]],
+            );
+            return row.revoked_at !== null;
+        };
+
+        const first = testJobContext();
+        await env.server.services.events.sweepStrandedWorkerSessions(first);
+        expect(await strandedRevoked()).toBe(false);
+        expect(first.saved.at(-1)).toMatchObject({
+            afterId: expect.any(Number),
+        });
+
+        const second = testJobContext(first.saved.at(-1));
+        await env.server.services.events.sweepStrandedWorkerSessions(second);
+        expect(await strandedRevoked()).toBe(true);
+        // The end of the table wraps the cursor back to the start.
+        expect(second.saved.at(-1)).toBeNull();
+    });
 });

src/backend/services/events/durable.integration.test.ts

@@ -872,6 +872,71 @@ describe('what a write costs once a durable row exists', () => {
 
         expect(deliveryOf(path)).toBeUndefined();
     });
+
+    it('publishes the expiry reap`s bump, so a region still caching the row drops it', async () => {
+        await clearRows();
+        const created = await subscribe(env.users.user.token, {
+            expiresAt: Math.floor(Date.now() / 1000) + 3600,
+        });
+        const subId = created.body.subId as string;
+        const stale =
+            await env.server.stores.durableSubscription.getBySubId(subId);
+        await env.server.clients.db.write(
+            `UPDATE \`${TABLE}\` SET \`expires_at\` = ? WHERE \`sub_id\` = ?`,
+            [Math.floor(Date.now() / 1000) - 1, subId],
+        );
+
+        const heard: Array<Record<string, unknown>> = [];
+        const listener = (_key: string, data: unknown) => {
+            heard.push(data as Record<string, unknown>);
+        };
+        const announced = vi.spyOn(
+            env.server.services.eventForward,
+            'announceGeneration',
+        );
+        env.server.clients.event.on(
+            'outer.pubsub.events.generationBumped',
+            listener as never,
+        );
+        try {
+            await expect(events().sweepExpired()).resolves.toBe(1);
+            // Over the addressed peer channel as well as the bus.
+            expect(announced).toHaveBeenCalledWith(
+                expect.objectContaining({ userId }),
+                true,
+                'subscription',
+            );
+        } finally {
+            env.server.clients.event.off(
+                'outer.pubsub.events.generationBumped',
+                listener as never,
+            );
+            announced.mockRestore();
+        }
+        const published = heard.find(
+            (bump) => bump.userId === userId && bump.durable === true,
+        );
+        expect(published).toBeDefined();
+
+        // Another region's Redis still holds the row; this one stands in for it.
+        await env.server.stores.eventSubscription.cacheDurable([stale!]);
+        delivered.length = 0;
+        const before = `${anchor}/stale-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: before });
+        await settle(before);
+        expect(deliveryOf(before)?.subId).toBe(subId);
+
+        await env.server.clients.event.emitAndWait(
+            'outer.pubsub.events.generationBumped',
+            published!,
+            { from_outside: true },
+        );
+        delivered.length = 0;
+        const after = `${anchor}/peer-swept-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: after });
+        await quiet();
+        expect(deliveryOf(after)).toBeUndefined();
+    });
 });
 
 describe('where a durable delivery is addressed', () => {

src/backend/services/events/fetch.integration.test.ts

@@ -616,6 +616,210 @@ describe('reading again from a saved cursor', () => {
     });
 });
 
+describe('starting from the end', () => {
+    const SUBJECT = 'notif:app-user';
+
+    const message = async (title: string, inApp = appUid): Promise<string> =>
+        (
+            await env.server.stores.notification.create({
+                userId,
+                value: { type: 'app.message', title },
+                type: 'app.message',
+                audience: 'app-user',
+                appUid: inApp,
+                group: `group-${uuidv4()}`,
+            })
+        ).uid as string;
+
+    const fromEnd = async (token = appToken): Promise<string> => {
+        const end = await fetchPage(token, { subject: SUBJECT, from: 'end' });
+        expect(end.status).toBe(200);
+        expect(end.body.items).toEqual([]);
+        expect(end.body.cursor).toBeTruthy();
+        return end.body.cursor!;
+    };
+
+    it('answers a cursor on an empty slice that returns what comes after', async () => {
+        await clearMailboxes();
+        const cursor = await fromEnd();
+
+        const later = await message('later');
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual([later]);
+    });
+
+    it('skips what is there and returns only newer rows', async () => {
+        await clearMailboxes();
+        for (let i = 0; i < 3; i++) await message(`before ${i}`);
+        const cursor = await fromEnd();
+
+        const quiet = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(quiet.body.items).toEqual([]);
+        expect(quiet.body.cursor).toBeUndefined();
+
+        const later = [await message('after 0'), await message('after 1')];
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual(later);
+    });
+
+    it('returns a row replaced after the cursor was taken', async () => {
+        await clearMailboxes();
+        const old = await message('old');
+        await message('older still unchanged');
+        const cursor = await fromEnd();
+
+        await env.server.services.notification.notifyUpdate(
+            old,
+            userId,
+            { title: 'replaced' },
+            { type: 'app.message', replace: true },
+        );
+
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual([old]);
+        expect(next.body.items![0]).toMatchObject({
+            replaced: true,
+            notification: { title: 'replaced' },
+        });
+    });
+
+    it('returns a replacement stamped by a clock behind the server`s', async () => {
+        await clearMailboxes();
+        const behind = Date.now() - 60_000;
+        const [early, late] = [await message('early'), await message('late')];
+        const replace = (uid: string, title: string, replacedAt: number) =>
+            env.server.stores.notification.updateValue(
+                uid,
+                userId,
+                { type: 'app.message', title },
+                { onlyUnseen: true, replacedAt },
+            );
+        await replace(early, 'replaced before', behind - 1_000);
+        const cursor = await fromEnd();
+
+        await replace(late, 'replaced after', behind);
+
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual([late]);
+        expect(next.body.items![0]).toMatchObject({
+            notification: { title: 'replaced after' },
+        });
+    });
+
+    it('keeps to the slice the subject names', async () => {
+        await clearMailboxes();
+        await message('own before');
+        const cursor = await fromEnd();
+
+        await message('other app', otherAppUid);
+        await seed(userId, { title: 'account row' });
+        const own = await message('own after');
+
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual([own]);
+    });
+
+    it('answers no cursor for a slice the caller cannot see', async () => {
+        await clearMailboxes();
+        await message('other app', otherAppUid);
+
+        for (const [token, subject] of [
+            [appToken, `notif:${otherAppUid}:app-user`],
+            [appToken, `notif:${userUuid}:account`],
+            [appReadUrlToken, SUBJECT],
+        ] as const) {
+            const page = await fetchPage(token, { subject, from: 'end' });
+            expect(page.status).toBe(200);
+            expect(page.body.items).toEqual([]);
+            expect(page.body.cursor).toBeUndefined();
+        }
+    });
+
+    it('reads from the cursor when one is given', async () => {
+        await clearMailboxes();
+        const uids = [await message('m0'), await message('m1')];
+        const first = await fetchPage(appToken, { subject: SUBJECT, limit: 1 });
+
+        const page = await fetchPage(appToken, {
+            subject: SUBJECT,
+            cursor: first.body.cursor,
+            from: 'end',
+        });
+        expect(uidsOf(page)).toEqual([uids[1]]);
+    });
+
+    it('rejects a `from` that is neither start nor end', async () => {
+        const page = await fetchPage(appToken, {
+            subject: SUBJECT,
+            from: 'middle',
+        });
+        expect(page.status).toBe(400);
+
+        const start = await fetchPage(appToken, {
+            subject: SUBJECT,
+            from: 'start',
+        });
+        expect(start.status).toBe(200);
+    });
+
+    it('starts the SDK feed without catch-up in one read', async () => {
+        await clearMailboxes();
+        for (let i = 0; i < 3; i++) await message(`before ${i}`);
+
+        const appActor = (await env.server.services.auth.authenticate(appToken))
+            .actor!;
+        const reads: Array<Record<string, unknown>> = [];
+        let receive: (delivery: { event: unknown }) => void = () => {};
+        const titles: string[] = [];
+        const feed = new NotificationFeed(
+            {
+                authToken: appToken,
+                events: {
+                    onLocal: async (
+                        _subject: string,
+                        handler: typeof receive,
+                    ) => {
+                        receive = handler;
+                        return { off: async () => {} };
+                    },
+                    fetch: async (options: Record<string, unknown>) => {
+                        reads.push(options);
+                        return env.server.services.events.fetchMissed(
+                            appActor,
+                            options,
+                        );
+                    },
+                },
+            } as never,
+            (notification: { title: string | null }) => {
+                titles.push(String(notification.title));
+            },
+            { catchUp: false },
+        );
+
+        try {
+            await feed.start();
+            expect(reads).toEqual([
+                { subject: SUBJECT, limit: 200, from: 'end' },
+            ]);
+
+            await message('after start');
+            receive({ event: { op: 'gap', reason: 'reconnect' } });
+            await vi.waitFor(() => expect(reads).toHaveLength(2));
+            await new Promise((resolve) => setTimeout(resolve, 0));
+        } finally {
+            await feed.off();
+        }
+
+        expect(reads[1]).toEqual({
+            subject: SUBJECT,
+            limit: 200,
+            cursor: expect.any(String),
+        });
+        expect(titles).toEqual(['after start']);
+    });
+});
+
 describe('cross-user isolation', () => {
     it('returns nothing of another account, whatever the subject names', async () => {
         await clearMailboxes();

src/backend/services/events/forwarding.test.ts

@@ -79,6 +79,14 @@ vi.mock(import('@opentelemetry/api'), async (importOriginal) => ({
                     metricCalls.push({ name, value, attributes });
                 },
             }),
+            createHistogram: (name: string) => ({
+                record: (
+                    value: number,
+                    attributes: Record<string, unknown> = {},
+                ) => {
+                    metricCalls.push({ name, value, attributes });
+                },
+            }),
         }),
     },
 }));
@@ -441,6 +449,7 @@ const makeRegion = (
             ): Promise<{
                 row: DurableSubscription;
                 bump: { userId: number; generation: number };
+                reaped: [];
             }> => {
                 const row = durableRow({
                     holderUserId: input.holderUserId as number,
@@ -469,6 +478,7 @@ const makeRegion = (
                             row.ownerUserId,
                         ),
                     },
+                    reaped: [],
                 };
             },
         },

src/backend/services/events/handlers.integration.test.ts

@@ -45,7 +45,10 @@ import {
     EVENTS_WORKER_SOURCE_MAX_BYTES,
 } from '../../controllers/events/limits.js';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
-import { hashContent } from '../../stores/events/EventHandlerStore.js';
+import {
+    appHandlerScope,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import type { DurableSubscriptionView } from './EventsService.js';
@@ -220,7 +223,8 @@ afterAll(async () => {
 
 beforeEach(async () => {
     await env.server.clients.db.write('DELETE FROM `event_handlers`', []);
-    for (const row of await durable().listActiveForHolder(userId, null))
+    for (const row of (await durable().listActiveForHolderPage(userId, null, 0))
+        .rows)
         await durable().remove(row);
     await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []);
     await env.server.stores.eventSubscription.rebuildDurable(userId, []);
@@ -531,7 +535,7 @@ describe('what a publish reads', () => {
                 if (sql.includes(BY_NAME_SQL) && !landed) {
                     landed = true;
                     await env.server.stores.eventHandler.publish({
-                        appUid,
+                        scope: appHandlerScope(appUid),
                         name: 'ingestUpload',
                         source: NEXT_SOURCE,
                         replace: true,
@@ -589,7 +593,7 @@ describe('what a publish reads', () => {
             pread.mockRestore();
         }
         const standing = await env.server.stores.eventHandler.getByName(
-            appUid,
+            appHandlerScope(appUid),
             'ingestUpload',
         );
         expect(standing?.sourceHash).toBe(hashContent(THIRD_SOURCE));
@@ -602,7 +606,7 @@ describe('what a publish reads', () => {
             if (sql.includes(BY_NAME_SQL) && !beaten) {
                 beaten = true;
                 await env.server.stores.eventHandler.publish({
-                    appUid,
+                    scope: appHandlerScope(appUid),
                     name: 'ingestUpload',
                     source: THIRD_SOURCE,
                 });
@@ -658,7 +662,7 @@ describe('what a publish reads', () => {
             pread.mockRestore();
         }
         await expect(
-            env.server.stores.eventHandler.countForApp(appUid),
+            env.server.stores.eventHandler.countForApp(appHandlerScope(appUid)),
         ).resolves.toBe(0);
     });
 
@@ -777,7 +781,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(404);
         expect(refused.body.code).toBe('events_handler_not_found');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('binds an inline body whose hash is what is published', async () => {
@@ -805,7 +811,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(409);
         expect(refused.body.code).toBe('events_handler_hash_mismatch');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('still needs a name for a subscription owed to one consumer', async () => {

src/backend/services/events/kvHandleRevoke.integration.test.ts

@@ -383,7 +383,7 @@ describe('revoking a handle', () => {
         const sub = await subscribeDurable(handle);
 
         const store = env.server.stores.durableSubscription;
-        const byHolder = vi.spyOn(store, 'listActiveForHolder');
+        const byHolder = vi.spyOn(store, 'listActiveForHolderPage');
         const settle = vi.spyOn(events(), 'settleRevokedGrant');
 
         try {
@@ -400,7 +400,9 @@ describe('revoking a handle', () => {
             for (const call of byHolder.mock.calls)
                 expect(call.slice(1)).toEqual([
                     null,
-                    { includeSuspended: true },
+                    0,
+                    undefined,
+                    { includeSuspended: true, excludeSchedules: true },
                 ]);
         } finally {
             byHolder.mockRestore();
@@ -442,7 +444,9 @@ describe('revoking a handle', () => {
         const page = await shares().listKvHandles(owner.actor, {
             limit: 100,
         });
-        const deeperRow = page.items.find((one) => one.handle === deeper.handle);
+        const deeperRow = page.items.find(
+            (one) => one.handle === deeper.handle,
+        );
         expect(deeperRow?.revokedAt).toBeTypeOf('number');
         expect((await countLive()).total).toBe(before.total);
     });

src/backend/services/events/metering.test.ts

@@ -36,13 +36,17 @@ import {
     DurableQuotaRaceLost,
     type DurableSubscriptionInput,
 } from '../../stores/events/DurableSubscriptionStore.js';
-import type { DurableSubscription } from '../../stores/events/types.js';
+import {
+    scheduleToken,
+    type DurableSubscription,
+} from '../../stores/events/types.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
 import {
     DEFAULT_FREE_SUBSCRIPTION,
     DEFAULT_TEMP_SUBSCRIPTION,
 } from '../metering/consts.js';
 import type { UsageInput } from '../metering/types.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_COSTS } from './costs.js';
 import { EventsService } from './EventsService.js';
@@ -62,6 +66,12 @@ let solvent: Set<number>;
 /** Set to have the next create lose the last slot after its insert. */
 let raceLost: boolean;
 let emitted: Array<{ key: string; data: unknown }>;
+/** Rows one credit-sweep page takes; null for the service's own batch. */
+let pageSize: number | null;
+/** Rows one holder page takes; null for all of them in one page. */
+let holderPageSize: number | null;
+/** What `services.schedules` hears about changed schedule rows. */
+let schedulesChanged: (rows: readonly DurableSubscription[]) => void;
 
 const anchorUid = (): string => `docs-${seq}`;
 const anchorPath = (): string => `/u${userId}/Documents`;
@@ -99,7 +109,10 @@ const actorFor = (over: Partial<Actor> = {}): Actor =>
     }) as unknown as Actor;
 
 const appActor = (appUid: string): Actor =>
-    actorFor({ app: { uid: appUid, id: 1 }, effectiveApp: { uid: appUid, id: 1 } });
+    actorFor({
+        app: { uid: appUid, id: 1 },
+        effectiveApp: { uid: appUid, id: 1 },
+    });
 
 const anchorEntry = (): FSEntry =>
     ({
@@ -124,6 +137,9 @@ beforeEach(() => {
     solvent = new Set([userId]);
     raceLost = false;
     emitted = [];
+    pageSize = null;
+    holderPageSize = null;
+    schedulesChanged = () => undefined;
 
     service = new EventsService(
         { events: { enabled: true } } as IConfig,
@@ -155,26 +171,49 @@ beforeEach(() => {
                     return {
                         row,
                         bump: { userId: input.ownerUserId, generation: 1 },
+                        reaped: [],
                     };
                 },
-                listSuspendedPage: async (reason: string, afterId: number) =>
-                    afterId === 0
-                        ? {
-                              rows: rows.filter(
-                                  (row) => row.suspendedReason === reason,
-                              ),
-                              nextId: null,
-                          }
-                        : { rows: [], nextId: null },
-                listSuspendedForHolder: async (
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedPage: async (
+                    reason: string,
+                    afterId: number,
+                    batchSize: number,
+                ) => {
+                    const size = pageSize ?? batchSize;
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId && row.suspendedReason === reason,
+                        )
+                        .slice(0, size);
+                    return {
+                        rows: page.map(({ row }) => row),
+                        nextId: page.length < size ? null : page.at(-1)!.id,
+                    };
+                },
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedForHolderPage: async (
                     holderUserId: number,
                     reason: string,
-                ) =>
-                    rows.filter(
-                        (row) =>
-                            row.holderUserId === holderUserId &&
-                            row.suspendedReason === reason,
-                    ),
+                    afterId: number,
+                ) => {
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId &&
+                                row.holderUserId === holderUserId &&
+                                row.suspendedReason === reason,
+                        );
+                    const size = holderPageSize ?? page.length;
+                    const taken = page.slice(0, size);
+                    return {
+                        rows: taken.map(({ row }) => row),
+                        nextId: page.length > size ? taken.at(-1)!.id : null,
+                    };
+                },
                 resume: async (resuming: readonly DurableSubscription[]) => {
                     for (const row of resuming) {
                         resumed.push(row.subId);
@@ -233,10 +272,7 @@ beforeEach(() => {
             // plan allows, not about who agreed to it.
             permission: { check: async () => true },
             metering: {
-                bufferIncrementUsages: (
-                    actor: Actor,
-                    usages: UsageInput[],
-                ) => {
+                bufferIncrementUsages: (actor: Actor, usages: UsageInput[]) => {
                     for (const usage of usages)
                         metered.push({
                             userUuid: actor.user?.uuid,
@@ -247,6 +283,11 @@ beforeEach(() => {
                     solvent.has(Number(actor.user?.id ?? -1)),
                 getActorSubscription: async () => ({ id: plan }),
             },
+            schedules: {
+                onSubscriptionsChanged: async (
+                    changed: readonly DurableSubscription[],
+                ) => schedulesChanged(changed),
+            },
         } as never,
     );
 });
@@ -258,23 +299,105 @@ describe('a suspension waiting on a balance', () => {
             suspendedReason: 'no_credit',
         });
 
-    it('comes back once the account has budget again', async () => {
+    it('comes back once the account has budget again, and every region hears it', async () => {
         const row = suspended();
         rows.push(row);
 
-        await expect(service.sweepNoCredit()).resolves.toBe(1);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(1);
         expect(resumed).toEqual([row.subId]);
         expect(row.suspendedReason).toBeNull();
+        expect(emitted).toContainEqual({
+            key: 'outer.pubsub.events.generationBumped',
+            data: { userId, generation: 1, durable: true },
+        });
     });
 
     it('stays out of service while the account still has none', async () => {
         rows.push(suspended());
         solvent.clear();
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 
+    it('reaches rows past one run`s page cap across runs, then wraps', async () => {
+        // One row per page, so a run's 200 pages stop at 200 rows.
+        pageSize = 1;
+        for (let i = 0; i < 450; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 10_000 + i;
+        // Only the last holder has a balance again.
+        solvent = new Set([10_000 + 449]);
+
+        const first = testJobContext();
+        await expect(service.sweepNoCredit(first)).resolves.toBe(0);
+        expect(first.saved.at(-1)).toEqual({ afterId: 200 });
+
+        const second = testJobContext(first.saved.at(-1));
+        await expect(service.sweepNoCredit(second)).resolves.toBe(0);
+        expect(second.saved.at(-1)).toEqual({ afterId: 400 });
+
+        const third = testJobContext(second.saved.at(-1));
+        await expect(service.sweepNoCredit(third)).resolves.toBe(1);
+        expect(resumed).toEqual([rows[449].subId]);
+        // The end of the table wraps the cursor back to the start.
+        expect(third.saved.at(-1)).toBeNull();
+    });
+
+    it('stops between pages once its signal aborts, keeping the last full page', async () => {
+        pageSize = 1;
+        for (let i = 0; i < 5; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 20_000 + i;
+        const ctx = testJobContext();
+        const save = ctx.saveState;
+        ctx.saveState = async (next) => {
+            await save(next);
+            if (ctx.saved.length === 2) ctx.abort();
+        };
+
+        await service.sweepNoCredit(ctx);
+        expect(ctx.saved).toEqual([{ afterId: 1 }, { afterId: 2 }]);
+    });
+
+    it('resumes no later holder page once its signal aborts inside a page`s hook, keeping the checkpoint', async () => {
+        // A full sweep page, so finishing it would move the checkpoint.
+        pageSize = 3;
+        holderPageSize = 1;
+        for (let i = 0; i < 3; i++)
+            rows.push(
+                durableRow({
+                    subId: `sched-${i}`,
+                    token: scheduleToken(`schedule-${seq}-${i}`),
+                    suspendedAt: Math.floor(Date.now() / 1000),
+                    suspendedReason: 'no_credit',
+                }),
+            );
+        const ctx = testJobContext();
+        const heard: string[][] = [];
+        schedulesChanged = (changed) => {
+            heard.push(changed.map((row) => row.subId));
+            ctx.abort();
+        };
+
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(1);
+        expect(resumed).toEqual(['sched-0']);
+        expect(heard).toEqual([['sched-0']]);
+        expect(rows[1].suspendedReason).toBe('no_credit');
+        expect(ctx.saved).toEqual([]);
+    });
+
+    it('wraps a saved cursor past the end of a shrunken table back to the start', async () => {
+        rows.push(suspended());
+        const ctx = testJobContext({ afterId: 999 });
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([null]);
+    });
+
+    it('writes no cursor while there is nothing to page through', async () => {
+        const ctx = testJobContext();
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([]);
+    });
+
     it('leaves a suspension it did not cause alone', async () => {
         rows.push(
             durableRow({
@@ -283,7 +406,7 @@ describe('a suspension waiting on a balance', () => {
             }),
         );
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 });
@@ -298,9 +421,10 @@ describe('what a plan lets an account subscribe to', () => {
         await subscribe(appActor(`app-${seq}`));
 
         expect(created[0].limits).toEqual({
-            perUser: EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
-                DEFAULT_FREE_SUBSCRIPTION
-            ],
+            perUser:
+                EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
+                    DEFAULT_FREE_SUBSCRIPTION
+                ],
             perApp: EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP.bySubscription[
                 DEFAULT_FREE_SUBSCRIPTION
             ],

src/backend/services/events/registry.test.ts

@@ -28,6 +28,7 @@ import {
     lookupFsSubject,
     lookupKvSubject,
     lookupPublicSubject,
+    projectScheduleTick,
     type FsDeliveryContext,
     type FsPublicSubject,
     type KvDeliveryContext,
@@ -324,3 +325,52 @@ describe('registry typing', () => {
         expect(subjects.length).toBeGreaterThan(0);
     });
 });
+
+describe('the schedule tick', () => {
+    const tick = {
+        scheduleId: '2f7c1b6e-4d1a-4c55-9a8e-0e6f3c2b1a90',
+        scheduledFor: 1_798_761_600_000,
+        missed: 0,
+        missedCapped: false,
+        attempt: 1,
+    };
+
+    it('carries exactly the fields a handler is promised', () => {
+        const event = projectScheduleTick(tick);
+        expect(Object.keys(event).sort()).toEqual([
+            'attempt',
+            'id',
+            'missed',
+            'op',
+            'scheduleId',
+            'scheduledFor',
+            'subject',
+        ]);
+        expect(event).toEqual({
+            subject: `schedule:${tick.scheduleId}`,
+            op: 'tick',
+            id: `${tick.scheduleId}:${tick.scheduledFor}`,
+            scheduleId: tick.scheduleId,
+            scheduledFor: tick.scheduledFor,
+            missed: 0,
+            attempt: 1,
+        });
+    });
+
+    it('says when `missed` stopped counting, and only then', () => {
+        const capped = projectScheduleTick({
+            ...tick,
+            missed: 1000,
+            missedCapped: true,
+        });
+        expect(capped.missedCapped).toBe(true);
+        expect(capped.missed).toBe(1000);
+        expect('missedCapped' in projectScheduleTick(tick)).toBe(false);
+    });
+
+    it('keeps the id across attempts at one slot', () => {
+        expect(projectScheduleTick({ ...tick, attempt: 3 }).id).toBe(
+            projectScheduleTick(tick).id,
+        );
+    });
+});

src/backend/services/events/registry.ts

@@ -20,6 +20,7 @@
 import type { EventKey, KvOp } from '../../clients/event/types.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
 import type { AclMode } from '../acl/ACLService.js';
+import { scheduleToken } from '../../stores/events/types.js';
 import type { NotificationAudience } from '../notification/notificationTypes.js';
 import { relativeTo } from './matcher.js';
 import {
@@ -359,6 +360,50 @@ const notifProject = (delivery: NotifDeliveryContext): ProjectedNotifEvent => ({
     seq: delivery.seq,
 });
 
+// -- Schedule ticks ---------------------------------------------------
+
+/**
+ * What a schedule's handler is handed as `event`. No bus event stands behind
+ * it, so it has no registry entry: the runner builds it from the slot it fires.
+ * Additive only, as for the other projections.
+ */
+export interface ScheduleEvent {
+    /** `schedule:<scheduleId>`. */
+    subject: string;
+    op: 'tick';
+    /** `${scheduleId}:${scheduledFor}`: the same across retries of one slot. */
+    id: string;
+    scheduleId: string;
+    /** The slot this run is for, in epoch ms. */
+    scheduledFor: number;
+    /** When the invocation was sent; stamped as it goes out. */
+    firedAt?: number;
+    /** Slots skipped since the last one that ran. */
+    missed: number;
+    /** Set when `missed` stopped counting at its cap. */
+    missedCapped?: true;
+    /** Attempts at this slot, this one included. */
+    attempt: number;
+}
+
+/** The {@link ScheduleEvent} of one run. Called by extensions. */
+export const projectScheduleTick = (tick: {
+    scheduleId: string;
+    scheduledFor: number;
+    missed: number;
+    missedCapped: boolean;
+    attempt: number;
+}): ScheduleEvent => ({
+    subject: scheduleToken(tick.scheduleId),
+    op: 'tick',
+    id: `${tick.scheduleId}:${tick.scheduledFor}`,
+    scheduleId: tick.scheduleId,
+    scheduledFor: tick.scheduledFor,
+    missed: tick.missed,
+    ...(tick.missedCapped ? { missedCapped: true as const } : {}),
+    attempt: tick.attempt,
+});
+
 // -- Registry ---------------------------------------------------------
 
 export const PUBLIC_SUBJECTS = [

src/backend/services/events/requestParsing.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { HttpError } from '../../core/http/HttpError.js';
+import { checkRateLimit } from '../../core/http/middleware/rateLimit.js';
+import type { RouteRateLimit } from '../../core/http/types.js';
+import {
+    HANDLER_NAME_MAX_LENGTH,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
+import type { HandlerScopeKind } from '../../stores/events/types.js';
+
+/** Request and response pieces the subscription and schedule verbs share. */
+
+export const badRequest = (message: string, code: string): HttpError =>
+    new HttpError(400, message, { legacyCode: code });
+
+export const parseHandlerName = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || value.length === 0)
+        throw badRequest('handlerName must be a string', 'bad_request');
+    if (value.length > HANDLER_NAME_MAX_LENGTH)
+        throw badRequest(
+            `handlerName may not exceed ${HANDLER_NAME_MAX_LENGTH} characters`,
+            'bad_request',
+        );
+    return value;
+};
+
+/** Hex digest of the inline source a subscribe claims it is binding. */
+export const parseHandlerHash = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value))
+        throw badRequest(
+            'handlerHash must be a sha-256 hex digest',
+            'bad_request',
+        );
+    return value;
+};
+
+/** Source of an inline handler; the store enforces the size and content rules. */
+export const parseHandlerSource = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || value.trim().length === 0)
+        throw badRequest(
+            'handlerSource must be a non-empty string',
+            'bad_request',
+        );
+    return value;
+};
+
+/** Which handler set a request is about; absent means the app's. */
+export const parseHandlerScopeKind = (
+    value: unknown,
+    field: string,
+): HandlerScopeKind => {
+    if (value === undefined || value === null || value === 'app') return 'app';
+    if (value === 'user') return 'user';
+    throw badRequest(`${field} must be 'app' or 'user'`, 'bad_request');
+};
+
+/** Stored as JSON text; the byte cap is the store's to enforce. */
+export const parseContext = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    try {
+        return JSON.stringify(value);
+    } catch {
+        throw badRequest('context must be JSON-serializable', 'bad_request');
+    }
+};
+
+const tooManyCalls = (): HttpError =>
+    new HttpError(429, 'Too many subscription changes', {
+        legacyCode: 'too_many_requests',
+    });
+
+/** Spend one call from a per-user window, refusing past it. */
+export const spendUserWindow = async (
+    limit: RouteRateLimit,
+    userId: number,
+): Promise<void> => {
+    const ok = await checkRateLimit(
+        `${limit.scope}:${userId}`,
+        limit.limit,
+        limit.window,
+    );
+    if (!ok) throw tooManyCalls();
+};
+
+/**
+ * A stored context as a listing may describe it: which keys it sets, and a hash
+ * of the whole blob. Never the values: the column is where an API key lives.
+ */
+export const projectContext = (
+    context: string | null,
+): { contextKeys: string[] | null; contextHash: string | null } => {
+    if (context === null) return { contextKeys: null, contextHash: null };
+    let keys: string[] = [];
+    try {
+        const parsed: unknown = JSON.parse(context);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
+            keys = Object.keys(parsed).sort();
+    } catch {
+        // Stored as JSON by the verbs, so this cannot normally happen; an
+        // unreadable blob still reports its hash rather than failing the list.
+    }
+    return { contextKeys: keys, contextHash: hashContent(context) };
+};

src/backend/services/events/revocationSettle.integration.test.ts

@@ -27,7 +27,15 @@
  * out of service so a revoked subscription stops costing anything at all.
  */
 
-import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import {
+    afterAll,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    onTestFinished,
+    vi,
+} from 'vitest';
 import { v4 as uuidv4 } from 'uuid';
 import { EVENTS_COALESCE_WINDOW_MS } from '../../controllers/events/limits.js';
 import { SUSPENDED_ROW_TTL_DAYS } from '../../controllers/events/limits.js';
@@ -43,6 +51,7 @@ import type { AclMode } from '../acl/ACLService.js';
 import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import type { DeliveryEnvelope } from './EventsService.js';
 import { fsAnchorToken } from './subjects.js';
+import { appHandlerScope } from '../../stores/events/EventHandlerStore.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 const TABLE = 'event_subscriptions';
@@ -206,7 +215,7 @@ const makeApp = async (path: string): Promise<Actor> => {
         EVENTS_BACKGROUND_PERMISSION,
     );
     await env.server.stores.eventHandler.publish({
-        appUid: uid,
+        scope: appHandlerScope(uid),
         name: 'onChange',
         source: 'async () => {}',
     });
@@ -950,3 +959,165 @@ describe('what a revoked grant settles', () => {
         }
     });
 });
+
+describe('a holder`s user scope when consent is withdrawn', () => {
+    const scopeOf = (appUid: string, holder = owner.id) => ({
+        appUid,
+        holder,
+    });
+    const putUserHandler = async (appUid: string, holder = owner.id) => {
+        await env.server.stores.eventHandler.publish({
+            scope: scopeOf(appUid, holder),
+            name: 'mine',
+            source: 'async () => {}',
+            broadcast: false,
+        });
+    };
+    const held = (appUid: string, holder = owner.id) =>
+        env.server.stores.eventHandler.listForApp(scopeOf(appUid, holder));
+
+    let destroys: Array<{ appUid: string; scope: string; userId: number }>;
+    beforeAll(() => {
+        env.server.clients.event.on('events.worker.destroy', (_key, event) => {
+            destroys?.push({
+                appUid: event.appUid,
+                scope: event.scope,
+                userId: event.actor.user.id,
+            });
+        });
+    });
+
+    it('drops the scope when background consent is revoked, announcing it as the holder', async () => {
+        await clearRows();
+        destroys = [];
+        const path = await folder(`/${owner.username}/scope-consent`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+
+        await env.server.services.permission.revokeUserAppPermission(
+            owner.actor,
+            app.uid,
+            EVENTS_BACKGROUND_PERMISSION,
+        );
+
+        await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
+            timeout: 5_000,
+            interval: 25,
+        });
+        expect(destroys.filter((event) => event.scope === 'user')).toEqual([
+            { appUid: app.uid, scope: 'user', userId: owner.id },
+        ]);
+    });
+
+    it('drops the scope when the whole app is revoked', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-whole-app`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+
+        await env.server.services.permission.revokeUserAppAll(
+            owner.actor,
+            app.uid,
+        );
+
+        await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
+            timeout: 5_000,
+            interval: 25,
+        });
+    });
+
+    it('leaves the scope when the consent still stands, as after a revoke then a grant', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-regrant`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+
+        for (const permission of [EVENTS_BACKGROUND_PERMISSION, null])
+            await events().settleRevokedGrant({
+                holderUserId: owner.id,
+                appUid: app.uid,
+                permission,
+            });
+
+        expect(await held(app.uid)).toHaveLength(1);
+    });
+
+    it('leaves the scope when the consent check itself fails', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-check-fails`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+        const check = vi
+            .spyOn(events(), 'hasBackgroundConsent')
+            .mockResolvedValue(null);
+        try {
+            await events().settleRevokedGrant({
+                holderUserId: owner.id,
+                appUid: app.uid,
+                permission: EVENTS_BACKGROUND_PERMISSION,
+            });
+            expect(check).toHaveBeenCalled();
+            expect(await held(app.uid)).toHaveLength(1);
+        } finally {
+            check.mockRestore();
+        }
+    });
+
+    it('leaves the scope for an unrelated grant revoked, and another holder`s scope for any', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-unrelated`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+        await putUserHandler(app.uid, guest.id);
+
+        await env.server.services.permission.revokeUserAppPermission(
+            owner.actor,
+            app.uid,
+            `fs:${await uidOf(path)}:list`,
+        );
+        await env.server.services.permission.revokeUserAppPermission(
+            owner.actor,
+            app.uid,
+            EVENTS_BACKGROUND_PERMISSION,
+        );
+
+        await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
+            timeout: 5_000,
+            interval: 25,
+        });
+        expect(await held(app.uid, guest.id)).toHaveLength(1);
+    });
+
+    it('keeps the rows bound to a dropped scope, suspended rather than deleted', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-rows`);
+        const appActor = await makeApp(path);
+        const app = appActor.app!;
+        const config = (events() as unknown as { config: IConfig }).config;
+        config.events!.userHandlers = true;
+        onTestFinished(() => {
+            config.events!.userHandlers = false;
+        });
+        const sub = (
+            await events().subscribeDurable(appActor, {
+                subject: `fs:${path}`,
+                handlerSource: 'async () => {}',
+            })
+        ).sub;
+
+        await env.server.services.permission.revokeUserAppPermission(
+            owner.actor,
+            app.uid,
+            EVENTS_BACKGROUND_PERMISSION,
+        );
+
+        await suspendedRow(sub.subId);
+        await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
+            timeout: 5_000,
+            interval: 25,
+        });
+        expect((await rowOf(sub.subId)).suspended_reason).toBe(
+            'permission_revoked',
+        );
+    });
+});

src/backend/services/events/scheduleSeam.integration.test.ts

@@ -0,0 +1,344 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The subscription lifecycle hands schedules' rows, and only theirs, to
+ * `services.schedules` with the change each path made. Without it every path
+ * still runs.
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type {
+    ScheduleServiceContract,
+    ScheduleSubscriptionChange,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import type { DurableSubscriptionInput } from '../../stores/events/DurableSubscriptionStore.js';
+import {
+    isScheduleRow,
+    scheduleToken,
+    type DurableSubscription,
+} from '../../stores/events/types.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
+
+const BOOT_TIMEOUT_MS = 120_000;
+const APP_UID = 'app-seam';
+
+let server: PuterServer;
+let userId: number;
+let anchorUid: string;
+let anchorPath: string;
+let saved: unknown;
+
+const durable = () => server.stores.durableSubscription;
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+
+const recording = () => {
+    const calls: Array<{
+        subIds: string[];
+        change: ScheduleSubscriptionChange;
+    }> = [];
+    const schedules = {
+        onSubscriptionsChanged: vi.fn(
+            async (
+                rows: readonly DurableSubscription[],
+                change: ScheduleSubscriptionChange,
+            ) => {
+                calls.push({ subIds: rows.map((r) => r.subId), change });
+            },
+        ),
+        createFor: vi.fn(),
+        list: vi.fn(),
+        get: vi.fn(),
+        remove: vi.fn(),
+        receivePeerChanges: vi.fn(),
+        prepareSendAt: vi.fn(),
+        createDeferredAppMessage: vi.fn(),
+    } satisfies ScheduleServiceContract;
+    setSchedules(schedules);
+    return calls;
+};
+
+const plainRow = async (
+    over: Partial<DurableSubscriptionInput> = {},
+): Promise<DurableSubscription> =>
+    (
+        await durable().create({
+            holderUserId: userId,
+            ownerUserId: userId,
+            appUid: APP_UID,
+            subject: `fs:${anchorPath}`,
+            token: `f#${anchorUid}`,
+            anchorUid,
+            anchorPath,
+            match: null,
+            op: null,
+            delivery: 'broadcast',
+            handlerName: 'onWrite',
+            context: null,
+            permission: 'list',
+            expiresAt: null,
+            ...over,
+        })
+    ).row;
+
+const scheduleRow = async (
+    over: Partial<DurableSubscriptionInput> = {},
+): Promise<DurableSubscription> => {
+    const uid = uuidv4();
+    return plainRow({
+        subject: scheduleToken(uid),
+        token: scheduleToken(uid),
+        anchorUid: uid,
+        anchorPath: '',
+        delivery: 'single',
+        handlerName: 'tick',
+        permission: 'events:background',
+        ...over,
+    });
+};
+
+const pendingCalls = () => {
+    const pending = server.stores.pendingDelivery;
+    return [
+        vi.spyOn(pending, 'purge'),
+        vi.spyOn(pending, 'hold'),
+        vi.spyOn(pending, 'releaseHold'),
+    ];
+};
+
+beforeAll(async () => {
+    server = await setupTestServer({ events: { enabled: true } } as IConfig);
+    const username = `seam-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+    });
+    userId = user.id;
+    anchorPath = `/${username}/seam`;
+    await server.services.fs.mkdir(userId, {
+        path: anchorPath,
+        createMissingParents: true,
+    });
+    anchorUid = (await server.stores.fsEntry.getEntryByPath(anchorPath))!.uid;
+    saved = slot().schedules;
+}, BOOT_TIMEOUT_MS);
+
+beforeEach(async () => {
+    await server.clients.db.write('DELETE FROM `event_subscriptions`', []);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+    vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+describe('the schedules lifecycle hook', () => {
+    it('gets only schedule rows on a suspend, and their backlog is never touched', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        const plain = await plainRow();
+        const [purge, hold, releaseHold] = pendingCalls();
+
+        await server.services.events.suspendSubscriptions(
+            [schedule],
+            'no_credit',
+        );
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'suspended', reason: 'no_credit' },
+            },
+        ]);
+        expect(purge).not.toHaveBeenCalled();
+        expect(hold).not.toHaveBeenCalled();
+
+        calls.length = 0;
+        await server.services.events.suspendSubscriptions([plain], 'no_credit');
+        expect(calls).toEqual([]);
+
+        calls.length = 0;
+        const suspended = (await durable().getBySubId(schedule.subId))!;
+        await server.services.events.resumeSubscriptions([suspended]);
+        expect(calls).toEqual([
+            { subIds: [schedule.subId], change: { type: 'resumed' } },
+        ]);
+        expect(releaseHold).not.toHaveBeenCalled();
+    });
+
+    it('ends schedules uninstalled when every grant to the app goes', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: null,
+        });
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: {
+                    type: 'suspended',
+                    reason: 'permission_revoked',
+                    endReason: 'uninstalled',
+                },
+            },
+        ]);
+    });
+
+    it('reads no schedule rows to settle a specific grant, and still settles them when consent goes', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        const plain = await plainRow();
+        const page = vi.spyOn(durable(), 'listActiveForHolderPage');
+        const readSubIds = async () =>
+            (await Promise.all(page.mock.results.map((r) => r.value))).flatMap(
+                (p) => p.rows.map((row: DurableSubscription) => row.subId),
+            );
+
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: `fs:${anchorUid}:list`,
+        });
+        const read = await readSubIds();
+        expect(read).toContain(plain.subId);
+        expect(read).not.toContain(schedule.subId);
+        expect(calls).toEqual([]);
+
+        page.mockClear();
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: EVENTS_BACKGROUND_PERMISSION,
+        });
+        expect(await readSubIds()).toContain(schedule.subId);
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'suspended', reason: 'permission_revoked' },
+            },
+        ]);
+    });
+
+    it('tells the holder once per kind when an uninstall settles several pages', async () => {
+        recording();
+        const plains = [await plainRow(), await plainRow(), await plainRow()];
+        const schedules = [await scheduleRow(), await scheduleRow()];
+        const store = durable();
+        const listPage = store.listActiveForHolderPage.bind(store);
+        vi.spyOn(store, 'listActiveForHolderPage').mockImplementation(
+            (holder, app, afterId, _batchSize, opts) =>
+                listPage(holder, app, afterId, 1, opts),
+        );
+        const notify = vi.spyOn(server.services.notification, 'notify');
+
+        await expect(
+            server.services.events.settleRevokedGrant({
+                holderUserId: userId,
+                appUid: APP_UID,
+                permission: null,
+            }),
+        ).resolves.toBe(5);
+        const ended = notify.mock.calls
+            .filter(([, , meta]) => meta?.type === 'app.events.ended')
+            .map(([, body]) => body as { count: number; subjects: string[] });
+        expect(ended).toHaveLength(2);
+        expect(ended.map((body) => body.count).sort()).toEqual([2, 3]);
+        expect(ended.flatMap((body) => body.subjects).sort()).toEqual(
+            [...plains, ...schedules].map((row) => row.subject).sort(),
+        );
+    });
+
+    it('ends schedules app_deleted when their app is deleted', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await server.services.events.settleDeletedApp(APP_UID);
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'ended', endReason: 'app_deleted' },
+            },
+        ]);
+    });
+
+    it('ends schedules whose rows expire', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow({ expiresAt: 1 });
+        await plainRow({ expiresAt: 1 });
+
+        await server.services.events.sweepExpired();
+        const handed = calls.flatMap((c) => c.subIds);
+        expect(handed).toEqual([schedule.subId]);
+        expect(calls[0].change).toEqual({ type: 'ended', endReason: 'ended' });
+    });
+
+    it('lets every path run without services.schedules', async () => {
+        setSchedules(undefined);
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await expect(
+            server.services.events.suspendSubscriptions(
+                [schedule],
+                'no_credit',
+            ),
+        ).resolves.toBe(1);
+        await expect(
+            server.services.events.resumeSubscriptions([
+                (await durable().getBySubId(schedule.subId))!,
+            ]),
+        ).resolves.toBe(1);
+        await server.services.events.settleDeletedApp(APP_UID);
+        const left = await server.clients.db.read(
+            'SELECT `token` FROM `event_subscriptions`',
+            [],
+        );
+        expect(
+            left.filter((r) => isScheduleRow(r as { token: string })),
+        ).toEqual([]);
+    });
+});

src/backend/services/events/userHandlers.worker.integration.test.ts

@@ -0,0 +1,386 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * A user-scope handler through the real local worker runtime: created inline by
+ * an app session, deployed eagerly as its holder, fired by an event, and run
+ * with the holder's identity. Also the rehydrate route's scope resolution.
+ */
+
+import http from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import {
+    appHandlerScope,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
+import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
+import { userHandlerScript } from './workerRuntime.js';
+
+const BOOT_TIMEOUT_MS = 180_000;
+const TEST_TIMEOUT_MS = 60_000;
+const INTERNAL_SECRET = 'events-internal-secret';
+
+const SOURCE = `async ({ event, ctx, user, fetch, ack }) => {
+    await fetch(ctx.sink, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({
+            label: ctx.label,
+            path: event.path,
+            token: user.authToken,
+        }),
+    });
+    await ack();
+}`;
+
+interface SinkPost {
+    label?: string;
+    path?: string;
+    token?: string;
+}
+
+let env: PuterTestEnv;
+let holders: Array<{ id: number; uuid: string; anchor: string; token: string }>;
+let appUid: string;
+let sink: http.Server;
+let sinkUrl: string;
+let sinkPosts: SinkPost[];
+
+const localWorkers = () => env.server.services.localworkerservice;
+
+const scriptFor = (source: string): string =>
+    userHandlerScript(
+        { name: `@${hashContent(source)}`, sourceHash: hashContent(source) },
+        '',
+    );
+
+const isResident = async (script: string): Promise<boolean> =>
+    (await localWorkers().dispatchEventsWorker(
+        new Request(`http://${script}.workers.puter.localhost/`, {
+            method: 'GET',
+        }),
+    )) !== null;
+
+const api = async (
+    method: 'GET' | 'POST',
+    path: string,
+    token: string,
+    body?: object,
+): Promise<{ status: number; body: Record<string, unknown> }> => {
+    const response = await fetch(new URL(path, env.apiOrigin), {
+        method,
+        headers: {
+            'content-type': 'application/json',
+            authorization: `Bearer ${token}`,
+        },
+        ...(body ? { body: JSON.stringify(body) } : {}),
+    });
+    return {
+        status: response.status,
+        body: (await response.json()) as Record<string, unknown>,
+    };
+};
+
+const rehydrate = async (
+    body: object,
+): Promise<{ status: number; body: Record<string, unknown> }> => {
+    const response = await fetch(
+        new URL('/events/worker/rehydrate', env.apiOrigin),
+        {
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json',
+                'x-puter-internal-auth': INTERNAL_SECRET,
+            },
+            body: JSON.stringify(body),
+        },
+    );
+    return {
+        status: response.status,
+        body: (await response.json()) as Record<string, unknown>,
+    };
+};
+
+const createInline = async (
+    holder: (typeof holders)[number],
+    label: string,
+    source = SOURCE,
+): Promise<string> => {
+    const response = await api('POST', '/events/subscribe', holder.token, {
+        subject: `fs:${holder.anchor}`,
+        delivery: 'single',
+        handlerSource: source,
+        context: { sink: sinkUrl, label },
+    });
+    expect(response.status).toBe(200);
+    return String(response.body.subId);
+};
+
+const delivered = (count = 1): Promise<void> =>
+    vi.waitFor(() => expect(sinkPosts.length).toBeGreaterThanOrEqual(count), {
+        timeout: 30_000,
+        interval: 50,
+    });
+
+beforeAll(async () => {
+    env = await setupPuterTestEnv({
+        events: {
+            enabled: true,
+            workerRuntime: true,
+            userHandlers: true,
+            internalSecret: INTERNAL_SECRET,
+        },
+        workers: { localServer: true },
+        unlimitedMetering: true,
+    } as IConfig);
+
+    const owner = (await env.server.stores.user.getByUsername(
+        env.users.user.username,
+    ))!;
+    appUid = `app-${uuidv4()}`;
+    await env.server.clients.db.write(
+        'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
+        [appUid, appUid, appUid, `https://${appUid}.example/`, owner.id],
+    );
+
+    holders = [];
+    for (const account of [env.users.user, env.users.other]) {
+        const user = (await env.server.stores.user.getByUsername(
+            account.username,
+        ))!;
+        const anchor = `/${account.username}/uh-e2e`;
+        await env.server.services.fs.mkdir(user.id, {
+            path: anchor,
+            createMissingParents: true,
+        });
+        const entry = await env.server.stores.fsEntry.getEntryByPath(anchor);
+        const { actor } = await env.server.services.auth.authenticate(
+            account.token,
+        );
+        for (const permission of [
+            `fs:${entry!.uid}:list`,
+            EVENTS_BACKGROUND_PERMISSION,
+        ])
+            await env.server.services.permission.grantUserAppPermission(
+                actor!,
+                appUid,
+                permission,
+            );
+        holders.push({
+            id: user.id,
+            uuid: user.uuid,
+            anchor,
+            token: await env.server.services.auth.getUserAppToken(
+                actor!,
+                appUid,
+            ),
+        });
+    }
+
+    sinkPosts = [];
+    sink = http.createServer((req, res) => {
+        const chunks: Buffer[] = [];
+        req.on('data', (chunk: Buffer) => chunks.push(chunk));
+        req.on('end', () => {
+            sinkPosts.push(
+                JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'),
+            );
+            res.writeHead(200).end();
+        });
+    });
+    await new Promise<void>((resolve) =>
+        sink.listen(0, '127.0.0.1', () => resolve()),
+    );
+    sinkUrl = `http://127.0.0.1:${(sink.address() as AddressInfo).port}`;
+}, BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+    await new Promise<void>((resolve) => {
+        if (!sink) return resolve();
+        sink.closeAllConnections();
+        sink.close(() => resolve());
+    });
+    await env?.shutdown();
+});
+
+beforeEach(() => {
+    sinkPosts = [];
+});
+
+describe('an inline handler in a user scope, end to end', () => {
+    it(
+        'is deployed eagerly, fires on an event and runs as its holder',
+        { timeout: TEST_TIMEOUT_MS },
+        async () => {
+            const [holder] = holders;
+            const source = `${SOURCE}\n// run-as-holder`;
+            const script = scriptFor(source);
+            expect(await isResident(script)).toBe(false);
+
+            await createInline(holder, 'inline', source);
+
+            // The deploy started with the publish, before any event.
+            await vi.waitFor(async () =>
+                expect(await isResident(script)).toBe(true),
+            );
+
+            await env.server.services.fs.touch(holder.id, {
+                path: `${holder.anchor}/note.txt`,
+            });
+            await delivered();
+
+            expect(sinkPosts[0]).toMatchObject({
+                label: 'inline',
+                path: `${holder.anchor}/note.txt`,
+            });
+            const { actor } = await env.server.services.auth.authenticate(
+                sinkPosts[0].token!,
+            );
+            expect(actor!.user?.id).toBe(holder.id);
+            expect(actor!.effectiveApp?.uid).toBe(appUid);
+        },
+    );
+
+    it(
+        'shares one deployed script between holders of the same code, each run as its own holder',
+        { timeout: TEST_TIMEOUT_MS },
+        async () => {
+            const source = `${SOURCE}\n// shared-code`;
+            const script = scriptFor(source);
+            const create = vi.spyOn(env.server.drivers.workers, 'create');
+
+            await createInline(holders[0], 'first', source);
+            await vi.waitFor(async () =>
+                expect(await isResident(script)).toBe(true),
+            );
+            await createInline(holders[1], 'second', source);
+            // Let the second holder's eager deploy settle before counting.
+            await new Promise((resolve) => setTimeout(resolve, 500));
+
+            // The marker made the second holder's eager deploy a no-op.
+            expect(
+                create.mock.calls.filter(
+                    ([args]) =>
+                        (args as { workerName: string }).workerName === script,
+                ),
+            ).toHaveLength(1);
+
+            for (const [i, holder] of holders.entries())
+                await env.server.services.fs.touch(holder.id, {
+                    path: `${holder.anchor}/shared-${i}.txt`,
+                });
+            await delivered(2);
+            const runAs = new Set<number>();
+            for (const post of sinkPosts)
+                runAs.add(
+                    (await env.server.services.auth.authenticate(post.token!))
+                        .actor!.user!.id,
+                );
+            expect(runAs).toEqual(new Set(holders.map((h) => h.id)));
+            create.mockRestore();
+        },
+    );
+});
+
+describe('the rehydrate route', () => {
+    const source = `${SOURCE}\n// rehydrate`;
+    const script = scriptFor(source);
+
+    beforeAll(async () => {
+        // Rows only: the handler is published, nothing is deployed here.
+        const eventHandler = env.server.stores.eventHandler;
+        await eventHandler.publish({
+            scope: { appUid, holder: holders[0].id },
+            name: `@${hashContent(source)}`,
+            source,
+            broadcast: false,
+        });
+        await eventHandler.publish({
+            scope: { appUid, holder: holders[1].id },
+            name: 'other',
+            source: SOURCE,
+            broadcast: false,
+        });
+    });
+
+    it('resolves the holder`s user scope when the body names one', async () => {
+        const response = await rehydrate({
+            script,
+            appUid,
+            holder: holders[0].uuid,
+        });
+        expect(response).toMatchObject({
+            status: 200,
+            body: { deployed: true },
+        });
+        expect(await isResident(script)).toBe(true);
+    });
+
+    it('reads an absent or empty holder as the app scope, which has no such handler', async () => {
+        for (const body of [
+            { script, appUid },
+            { script, appUid, holder: '' },
+        ])
+            expect(await rehydrate(body)).toMatchObject({
+                status: 404,
+                body: { deployed: false, reason: 'no-handlers' },
+            });
+        expect(
+            await env.server.stores.eventHandler.listForApp(
+                appHandlerScope(appUid),
+            ),
+        ).toEqual([]);
+    });
+
+    it('answers stale for a holder whose scope does not hold that handler', async () => {
+        expect(
+            await rehydrate({ script, appUid, holder: holders[1].uuid }),
+        ).toMatchObject({
+            status: 404,
+            body: { deployed: false, reason: 'stale' },
+        });
+    });
+
+    it('refuses a holder that does not exist, and a malformed one', async () => {
+        expect(
+            await rehydrate({ script, appUid, holder: uuidv4() }),
+        ).toMatchObject({
+            status: 404,
+            body: { deployed: false, reason: 'no-holder' },
+        });
+        expect(
+            (await rehydrate({ script, appUid, holder: 'a b' })).status,
+        ).toBe(400);
+        expect((await rehydrate({ script, appUid, holder: 7 })).status).toBe(
+            400,
+        );
+    });
+});

src/backend/services/events/workerInvoker.integration.test.ts

@@ -34,6 +34,7 @@ import type { AddressInfo } from 'node:net';
 import { v4 as uuidv4 } from 'uuid';
 import {
     afterAll,
+    afterEach,
     beforeAll,
     beforeEach,
     describe,
@@ -58,8 +59,14 @@ import {
     EVENTS_WORKER_SESSION_NAME,
     eventsInvokeKey,
     eventsWorkerScript,
+    userHandlerScript,
 } from './workerRuntime.js';
 import { handlerSetHash } from './workerSource.js';
+import {
+    appHandlerScope,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
+import type { WorkerInvocation } from './workerSeam.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 /** Short enough that a hung handler is a test case rather than a stall. */
@@ -129,6 +136,7 @@ const startStub = async (): Promise<string> => {
                     script: req.headers['x-puter-events-script'] as string,
                     app: req.headers['x-puter-events-app'] as string,
                     key: req.headers['x-puter-events-key'] as string,
+                    holder: req.headers['x-puter-events-holder'] as string,
                     deployed,
                 },
                 body: JSON.parse(raw || '{}') as StubCall['body'],
@@ -233,7 +241,7 @@ const makeWorkerApp = async (): Promise<{
     );
     const token = await env.server.services.auth.getUserAppToken(actor!, uid);
     await env.server.stores.eventHandler.publish({
-        appUid: uid,
+        scope: appHandlerScope(uid),
         name: HANDLER,
         source: SOURCE,
     });
@@ -403,7 +411,7 @@ beforeAll(async () => {
     );
     appToken = await env.server.services.auth.getUserAppToken(actor!, appUid);
     await env.server.stores.eventHandler.publish({
-        appUid,
+        scope: appHandlerScope(appUid),
         name: HANDLER,
         source: SOURCE,
     });
@@ -471,7 +479,9 @@ describe('the call an owed delivery makes', () => {
 
         const script = eventsWorkerScript(
             handlerSetHash(
-                await env.server.stores.eventHandler.setForApp(appUid),
+                await env.server.stores.eventHandler.setForApp(
+                    appHandlerScope(appUid),
+                ),
             ),
             // No `workers.internetExposedUrl` in this test's config.
             '',
@@ -1014,7 +1024,10 @@ describe('with no events worker to address', () => {
         const subId = await subscribe();
         // Unpublished after the fact: the app's set now hashes to nothing, so
         // there is no script to name and nowhere to send the delivery.
-        await env.server.stores.eventHandler.remove(appUid, HANDLER);
+        await env.server.stores.eventHandler.remove(
+            appHandlerScope(appUid),
+            HANDLER,
+        );
 
         try {
             await touch('unresolved.txt');
@@ -1037,7 +1050,7 @@ describe('with no events worker to address', () => {
             expect(calls).toEqual([]);
         } finally {
             await env.server.stores.eventHandler.publish({
-                appUid,
+                scope: appHandlerScope(appUid),
                 name: HANDLER,
                 source: SOURCE,
             });
@@ -1416,3 +1429,349 @@ describe('handlers whose writes run handlers', () => {
         ).resolves.toBeNull();
     });
 });
+
+describe('a handler in a user scope', () => {
+    const USER_SOURCE =
+        'async ({ event, ctx }) => { console.log("user", event.path, ctx.label); }';
+    const userName = `@${hashContent(USER_SOURCE)}`;
+    let holderUuid: string;
+    let createSpy: ReturnType<typeof vi.spyOn>;
+    let meterSpy: ReturnType<typeof vi.spyOn>;
+
+    const setFlag = (on: boolean): void => {
+        (
+            events() as unknown as { config: IConfig }
+        ).config.events!.userHandlers = on;
+    };
+    const userScope = () => ({ appUid, holder: userId });
+    const userScript = () =>
+        userHandlerScript(
+            { name: userName, sourceHash: hashContent(USER_SOURCE) },
+            '',
+        );
+
+    const post = async (path: string, body: object) => {
+        const response = await fetch(new URL(path, env.apiOrigin), {
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json',
+                authorization: `Bearer ${appToken}`,
+            },
+            body: JSON.stringify(body),
+        });
+        return {
+            status: response.status,
+            body: (await response.json()) as Record<string, unknown>,
+        };
+    };
+    /** A single-delivery row bound to a handler published by its own source. */
+    const inlineRow = async (source = USER_SOURCE): Promise<string> => {
+        const created = await post('/events/subscribe', {
+            subject: `fs:${anchor}`,
+            delivery: 'single',
+            handlerSource: source,
+            context: { label: 'user' },
+        });
+        expect(created.status).toBe(200);
+        return String(created.body.subId);
+    };
+    const invocationFor = (subId: string): WorkerInvocation => ({
+        subId,
+        holderUserId: userId,
+        appUid,
+        handlerName: userName,
+        handlerScope: 'user',
+        event: {
+            subject: 'schedule:s1',
+            op: 'tick',
+            id: 's1:1',
+            scheduleId: 's1',
+            scheduledFor: 1,
+            missed: 0,
+            attempt: 1,
+        },
+        context: null,
+        handlerDepth: 0,
+    });
+
+    beforeAll(async () => {
+        holderUuid = (await env.server.stores.user.getById(userId))!.uuid;
+    });
+    beforeEach(async () => {
+        setFlag(true);
+        await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
+        // Eager and on-miss deploys have no deploy target in this suite.
+        createSpy = vi
+            .spyOn(env.server.drivers.workers, 'create')
+            .mockResolvedValue({ success: true, errors: [] });
+        meterSpy = vi.spyOn(
+            env.server.services.metering,
+            'bufferIncrementUsages',
+        );
+    });
+    afterEach(async () => {
+        createSpy.mockRestore();
+        meterSpy.mockRestore();
+        setFlag(false);
+        await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
+    });
+
+    it('addresses the handler`s own script and names its holder', async () => {
+        const subId = await inlineRow();
+        await touch('user-scope.txt');
+        await invoked(1);
+
+        const call = calls[0];
+        expect(call.headers.script).toBe(userScript());
+        expect(call.headers.holder).toBe(holderUuid);
+        expect(call.headers.key).toBe(
+            eventsInvokeKey(INTERNAL_SECRET, userScript()),
+        );
+        expect(call.body.handler).toBe(userName);
+        await waitUntil(async () =>
+            expect(await pending().depth(subId)).toBe(0),
+        );
+    });
+
+    it('sends no holder for an app-scope row', async () => {
+        await subscribe();
+        await touch('app-scope.txt');
+        await invoked(1);
+        expect(calls[0].headers.holder).toBeUndefined();
+    });
+
+    it('keeps a user row and an app row of the same name on their own scope`s handler', async () => {
+        await env.server.stores.eventHandler.publish({
+            scope: userScope(),
+            name: HANDLER,
+            source: USER_SOURCE,
+        });
+        const created = await post('/events/subscribe', {
+            subject: `fs:${anchor}`,
+            delivery: 'single',
+            handlerName: HANDLER,
+            handlerScope: 'user',
+        });
+        expect(created.status).toBe(200);
+        await touch('same-name-user.txt');
+        await invoked(1);
+        // One at a time: a second row firing alongside would go as a batch.
+        expect(
+            (
+                await post('/events/unsubscribe', {
+                    subId: String(created.body.subId),
+                })
+            ).status,
+        ).toBe(200);
+        await subscribe();
+        await touch('same-name-app.txt');
+        await invoked(2);
+
+        const appScript = eventsWorkerScript(
+            handlerSetHash(
+                await env.server.stores.eventHandler.setForApp(
+                    appHandlerScope(appUid),
+                ),
+            ),
+            '',
+        );
+        const mine = userHandlerScript(
+            { name: HANDLER, sourceHash: hashContent(USER_SOURCE) },
+            '',
+        );
+        expect(mine).not.toBe(appScript);
+        expect(calls[0].headers).toMatchObject({
+            script: mine,
+            holder: holderUuid,
+        });
+        expect(calls[1].headers.script).toBe(appScript);
+        expect(calls[1].headers.holder).toBeUndefined();
+    });
+
+    it('suspends a row whose handler left its holder`s set, and tells the holder only', async () => {
+        const notify = vi.spyOn(env.server.services.notification, 'notify');
+        const subId = await inlineRow();
+        // Gone without the removal path that would suspend its rows.
+        await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
+
+        await touch('missing.txt');
+        await waitUntil(async () =>
+            expect((await durable().getBySubId(subId))?.suspendedReason).toBe(
+                'handler_not_found',
+            ),
+        );
+        expect(calls).toEqual([]);
+        expect(notify).toHaveBeenCalledWith(
+            [userId],
+            expect.objectContaining({ handler: userName }),
+            expect.objectContaining({
+                type: 'app.events.handler_suspended',
+                appUid,
+            }),
+        );
+        expect(notify).not.toHaveBeenCalledWith(
+            expect.anything(),
+            expect.anything(),
+            expect.objectContaining({ type: 'app.events.suspended' }),
+        );
+        notify.mockRestore();
+    });
+
+    it('does not suspend a schedule run whose replica read lags the publish', async () => {
+        const subId = await inlineRow();
+        const store = env.server.stores.eventHandler;
+        const real = store.setForApp.bind(store);
+        // The replica has not seen the insert; the primary has.
+        const lagging = vi
+            .spyOn(store, 'setForApp')
+            .mockImplementation(async (scope, options) =>
+                options?.replica
+                    ? [{ name: 'other', sourceHash: 'a'.repeat(64) }]
+                    : real(scope, options),
+            );
+        try {
+            const result = await events().invokeForSchedule(
+                invocationFor(subId),
+            );
+            expect(result).toMatchObject({ outcome: 'settled' });
+            expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+            expect(calls[0].headers.script).toBe(userScript());
+        } finally {
+            lagging.mockRestore();
+        }
+    });
+
+    it('reads a cold user scope from the replica only, once, for a delivery', async () => {
+        await inlineRow();
+        // Eager deploy at publish has finished with the primary by now.
+        await new Promise((resolve) => setTimeout(resolve, 300));
+        const redis = env.server.clients.redis;
+        const key = `eventHandlers:uset:${appUid}:${userId}`;
+        await redis.del(key, `${key}:r`);
+        const db = env.server.clients.db;
+        const primary = vi.spyOn(db, 'pread');
+        const replica = vi.spyOn(db, 'read');
+        const handlerReads = (spy: typeof primary) =>
+            spy.mock.calls.filter(([sql]) =>
+                String(sql).includes('event_user_handlers'),
+            ).length;
+
+        await touch('cold-user-scope.txt');
+        await invoked(1);
+
+        expect(handlerReads(primary)).toBe(0);
+        expect(handlerReads(replica)).toBe(1);
+        primary.mockRestore();
+        replica.mockRestore();
+    });
+
+    it('resumes a suspended row when the name is published again', async () => {
+        const subId = await inlineRow();
+        await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
+        await touch('gone.txt');
+        await waitUntil(async () =>
+            expect((await durable().getBySubId(subId))?.suspendedReason).toBe(
+                'handler_not_found',
+            ),
+        );
+
+        expect(
+            (
+                await post('/events/handlers/publish', {
+                    scope: 'user',
+                    name: userName,
+                    source: USER_SOURCE,
+                })
+            ).status,
+        ).toBe(200);
+        expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+    });
+
+    it('does not address a holder that is suspended, and leaves the row owed', async () => {
+        const subId = await inlineRow();
+        await env.server.clients.db.write(
+            'UPDATE user SET suspended = 1 WHERE id = ?',
+            [userId],
+        );
+        await env.server.stores.user.invalidateById(userId);
+        try {
+            await touch('suspended-holder.txt');
+            await failuresRecorded(subId, 1);
+            expect(calls).toEqual([]);
+            expect(await pending().depth(subId)).toBe(1);
+            expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+        } finally {
+            await env.server.clients.db.write(
+                'UPDATE user SET suspended = 0 WHERE id = ?',
+                [userId],
+            );
+            await env.server.stores.user.invalidateById(userId);
+        }
+    });
+
+    it('deploys the holder`s script on a dispatcher miss, as that holder', async () => {
+        dispatchMissing = true;
+        await inlineRow();
+        await touch('miss.txt');
+        await invoked(2);
+
+        expect(calls[0].headers.deployed).toBeUndefined();
+        expect(calls[1].headers.deployed).toBe('1');
+        // Once for the eager deploy at publish, once for the miss.
+        const deployed = createSpy.mock.calls.filter(
+            ([args]) =>
+                (args as { workerName: string }).workerName === userScript(),
+        );
+        expect(deployed.length).toBeGreaterThanOrEqual(1);
+    });
+
+    describe('with the flag off', () => {
+        it('leaves the delivery owed: nothing is called, suspended or billed', async () => {
+            const subId = await inlineRow();
+            setFlag(false);
+            meterSpy.mockClear();
+
+            await touch('flag-off.txt');
+            await new Promise((resolve) => setTimeout(resolve, 400));
+
+            expect(calls).toEqual([]);
+            expect(await pending().depth(subId)).toBe(1);
+            expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+            expect(meterSpy).not.toHaveBeenCalled();
+        });
+
+        it('answers unavailable from the invoker and for a schedule run', async () => {
+            const subId = await inlineRow();
+            setFlag(false);
+            const invocation = invocationFor(subId);
+
+            const worker = events().worker as unknown as {
+                run: (i: WorkerInvocation) => Promise<unknown>;
+            };
+            expect(await worker.run(invocation)).toEqual({
+                outcome: 'unavailable',
+            });
+            expect(await events().invokeForSchedule(invocation)).toEqual({
+                outcome: 'unavailable',
+            });
+            expect(calls).toEqual([]);
+            expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+        });
+
+        it('leaves an app-scope row alone', async () => {
+            setFlag(false);
+            await subscribe();
+            await touch('app-row-flag-off.txt');
+            await invoked(1);
+            expect(calls[0].headers.script).toMatch(/^evw-/);
+        });
+    });
+
+    it('runs a schedule invocation of a user scope with its holder', async () => {
+        const subId = await inlineRow();
+        const result = await events().invokeForSchedule(invocationFor(subId));
+        expect(result).toMatchObject({ outcome: 'settled' });
+        expect(calls[0].headers.script).toBe(userScript());
+        expect(calls[0].headers.holder).toBe(holderUuid);
+    });
+});

src/backend/services/events/workerRuntime.integration.test.ts

@@ -61,6 +61,7 @@ import {
     eventsWorkerScript,
 } from './workerRuntime.js';
 import { handlerSetHash } from './workerSource.js';
+import { appHandlerScope } from '../../stores/events/EventHandlerStore.js';
 
 const BOOT_TIMEOUT_MS = 180_000;
 const TEST_TIMEOUT_MS = 60_000;
@@ -171,7 +172,7 @@ const localWorkers = () => env.server.services.localworkerservice;
 /** The script the app's published set currently deploys as. */
 const scriptName = async (): Promise<string> =>
     eventsWorkerScript(
-        handlerSetHash(await env.server.stores.eventHandler.setForApp(appUid)),
+        handlerSetHash(await env.server.stores.eventHandler.setForApp(appHandlerScope(appUid))),
         // No `workers.internetExposedUrl` in this test's config, so the scope
         // this backend binds is empty.
         '',
@@ -886,7 +887,7 @@ describe('the worker follows the handler set', () => {
             // An empty set names no script, so nothing can be deployed for
             // this app again until something is published.
             expect(
-                await env.server.stores.eventHandler.setForApp(appUid),
+                await env.server.stores.eventHandler.setForApp(appHandlerScope(appUid)),
             ).toEqual([]);
             expect((await durable().getBySubId(subId))?.suspendedReason).toBe(
                 'handler_not_found',

src/backend/services/events/workerRuntime.test.ts

@@ -23,7 +23,9 @@ import {
     eventsInvokeKey,
     eventsWorkerScope,
     eventsWorkerScript,
+    userHandlerScript,
 } from './workerRuntime.js';
+import { handlerSetHash } from './workerSource.js';
 
 const SET_HASH = 'a1'.repeat(32);
 const OTHER_HASH = 'b2'.repeat(32);
@@ -50,6 +52,37 @@ describe('eventsWorkerScript', () => {
     });
 });
 
+describe('userHandlerScript', () => {
+    const handler = { name: '@abc', sourceHash: SET_HASH };
+
+    it('is the script of the handler as a set of one', () => {
+        expect(userHandlerScript(handler, '')).toBe(
+            eventsWorkerScript(handlerSetHash([handler]), ''),
+        );
+    });
+
+    it('is the same for identical code whoever holds it, and differs by code and name', () => {
+        // Nothing but name and source hash goes in: two holders (or two apps)
+        // of the same handler deploy one script.
+        const again = { ...handler };
+        expect(userHandlerScript(again, 's')).toBe(
+            userHandlerScript(handler, 's'),
+        );
+        expect(
+            userHandlerScript({ ...handler, sourceHash: OTHER_HASH }, 's'),
+        ).not.toBe(userHandlerScript(handler, 's'));
+        expect(userHandlerScript({ ...handler, name: '@abd' }, 's')).not.toBe(
+            userHandlerScript(handler, 's'),
+        );
+    });
+
+    it('stays scoped to the backend', () => {
+        expect(userHandlerScript(handler, 'https://a.example')).not.toBe(
+            userHandlerScript(handler, 'https://b.example'),
+        );
+    });
+});
+
 describe('eventsWorkerScope', () => {
     it('reads the same field `#cfDeploy` binds as `puter_endpoint`, and falls back to empty', () => {
         expect(eventsWorkerScope({})).toBe('');

src/backend/services/events/workerRuntime.ts

@@ -18,6 +18,8 @@
  */
 
 import { createHash, createHmac } from 'node:crypto';
+import type { EventHandler } from '../../stores/events/EventHandlerStore.js';
+import { handlerSetHash } from './workerSource.js';
 
 /**
  * How an app's events worker is named and how it recognizes the platform.
@@ -71,6 +73,17 @@ export const eventsWorkerScript = (setHash: string, scope: string): string =>
         .digest('hex')
         .slice(0, 32)}`;
 
+/**
+ * The script one user-scope handler deploys as: the set-of-one name, built from
+ * the handler's name and source hash alone. The holder and the app are not part
+ * of it, so identical code under one name is one script however many users and
+ * apps publish it.
+ */
+export const userHandlerScript = (
+    handler: Pick<EventHandler, 'name' | 'sourceHash'>,
+    scope: string,
+): string => eventsWorkerScript(handlerSetHash([handler]), scope);
+
 /** Version prefix on a derived key, so a rotation is visible in a log line. */
 export const EVENTS_INVOKE_KEY_VERSION = 'k1';
 

src/backend/services/events/workerSeam.ts

@@ -17,8 +17,15 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import type { EventsWorkerInvokerClient } from '../../clients/events/EventsWorkerInvokerClient.js';
-import type { DeliverableEvent } from './registry.js';
+import type {
+    EventsWorkerInvokerClient,
+    WorkerInvokeOptions,
+} from '../../clients/events/EventsWorkerInvokerClient.js';
+import type { Actor } from '../../core/actor.js';
+import type { HandlerScopeKind } from '../../stores/events/types.js';
+import type { DeliverableEvent, ScheduleEvent } from './registry.js';
+
+export type { ScheduleEvent } from './registry.js';
 
 /**
  * Where a delivery leaves the event system for the app's own code.
@@ -36,11 +43,15 @@ export interface WorkerInvocation {
     /** The app whose handler runs, or null for an account-owned row. */
     appUid: string | null;
     handlerName: string | null;
-    event: DeliverableEvent;
+    /** Whose handler set `handlerName` is in: absent means the app's own. */
+    handlerScope?: HandlerScopeKind;
+    event: DeliverableEvent | ScheduleEvent;
     /** The subscription's stored context, delivered to the handler as `ctx`. */
     context: string | null;
     /** Handler runs behind `event`; the token this run is handed is one deeper. */
     handlerDepth: number;
+    /** The holder acting as the app, when the caller resolved it already. */
+    actor?: Actor;
 }
 
 /**
@@ -64,6 +75,12 @@ export interface WorkerInvokerSeam {
     available?(): boolean;
 }
 
+/**
+ * Why no token could be minted: nobody to run as, no consent to, or the mint
+ * itself failed.
+ */
+export type MintRefusal = 'no-identity' | 'no-consent' | 'mint-failed';
+
 /**
  * Mints the subscriber's app-under-user token for one invocation. The
  * `events:background` consent is what authorizes it, not a narrower
@@ -71,15 +88,35 @@ export interface WorkerInvokerSeam {
  */
 export type SubscriberTokenMinter = (
     invocation: WorkerInvocation,
-) => Promise<string | null>;
+) => Promise<{ token: string } | { refused: MintRefusal }>;
+
+/** Where the handler set is read from when its cache misses. */
+export interface AddressOptions {
+    missRead?: 'primary' | 'replica';
+    /** A handler the set must have; a set without it is re-read on the primary. */
+    handlerName?: string;
+    /** Absent means the app's own set. */
+    handlerScope?: HandlerScopeKind;
+    /** The user whose scope it is, for `handlerScope: 'user'`. */
+    holderUserId?: number;
+}
 
 /**
- * The script an app's handlers deploy as and its invocation key; null when the
- * app has no handlers or no events secret is configured.
+ * The script a handler deploys as and its invocation key; null when it has none
+ * to run, no events secret is configured, or (in a user scope) the holder is
+ * gone or suspended. `holder` is the holder's uuid, for a user scope.
  */
 export type EventsWorkerAddresser = (
     appUid: string,
-) => Promise<{ script: string; key: string } | null>;
+    options?: AddressOptions,
+) => Promise<{ script: string; key: string; holder?: string } | null>;
+
+/** What one run came to, including why nothing was invoked. */
+export type WorkerRunResult =
+    WorkerInvocationResult | { outcome: MintRefusal | 'deploy-busy' };
+
+/** How {@link EventsWorkerInvoker.run} reaches the worker. */
+export type WorkerRunOptions = WorkerInvokeOptions & AddressOptions;
 
 /** How often the same script's retriable failures are logged. */
 const WARN_INTERVAL_MS = 60_000;
@@ -97,6 +134,7 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
     readonly #mintToken: SubscriberTokenMinter;
     readonly #address: EventsWorkerAddresser;
     readonly #available: () => boolean;
+    readonly #userHandlersEnabled: () => boolean;
     /**
      * Script -> when it was last logged, so a failing script floods once a
      * minute.
@@ -108,11 +146,13 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
         mintToken: SubscriberTokenMinter,
         address: EventsWorkerAddresser,
         available: () => boolean,
+        userHandlersEnabled: () => boolean = () => true,
     ) {
         this.#client = client;
         this.#mintToken = mintToken;
         this.#address = address;
         this.#available = available;
+        this.#userHandlersEnabled = userHandlersEnabled;
     }
 
     available(): boolean {
@@ -122,28 +162,65 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
     async invoke(
         invocation: WorkerInvocation,
     ): Promise<WorkerInvocationResult> {
+        const result = await this.run(invocation);
+        switch (result.outcome) {
+            // Nothing was attempted.
+            case 'no-identity':
+            case 'no-consent':
+            case 'mint-failed':
+                return { outcome: 'deferred' };
+            case 'deploy-busy':
+                return { outcome: 'unavailable' };
+            default:
+                return result;
+        }
+    }
+
+    /** {@link invoke}, saying why nothing was invoked. */
+    async run(
+        invocation: WorkerInvocation,
+        options: WorkerRunOptions = {},
+    ): Promise<WorkerRunResult> {
         const { appUid, handlerName } = invocation;
         if (!appUid || !handlerName) return { outcome: 'deferred' };
         if (!this.#available()) return { outcome: 'unavailable' };
+        // Owed, not failed: nothing resolves a user scope while it is off.
+        const { handlerScope } = invocation;
+        if (handlerScope === 'user' && !this.#userHandlersEnabled())
+            return { outcome: 'unavailable' };
 
-        const token = await this.#mintToken(invocation);
-        if (token === null) return { outcome: 'deferred' };
+        const minted = await this.#mintToken(invocation);
+        if ('refused' in minted) return { outcome: minted.refused };
 
         // Nowhere to send it: the app has no handlers or no owner to run them
         // as. Retrying cannot fix that, so the consecutive-failure rule is what
         // stops it.
-        const address = await this.#address(appUid);
+        const address = await this.#address(appUid, {
+            missRead: options.missRead,
+            handlerName,
+            ...(handlerScope === 'user'
+                ? { handlerScope, holderUserId: invocation.holderUserId }
+                : {}),
+        });
         if (address === null) return { outcome: 'retriable' };
 
-        const result = await this.#client.invoke({
-            script: address.script,
-            key: address.key,
-            appUid,
-            handler: handlerName,
-            token,
-            event: invocation.event,
-            ctx: parseContext(invocation.context),
-        });
+        const result = await this.#client.invoke(
+            {
+                script: address.script,
+                key: address.key,
+                appUid,
+                ...(address.holder ? { holder: address.holder } : {}),
+                handler: handlerName,
+                token: minted.token,
+                event: invocation.event,
+                ctx: parseContext(invocation.context),
+            },
+            {
+                lane: options.lane,
+                stampEvent: options.stampEvent,
+            },
+        );
+        if (result.deployBusy) return { outcome: 'deploy-busy' };
         if (
             (result.outcome === 'retriable' ||
                 result.outcome === 'unavailable') &&

src/backend/services/events/workers.integration.test.ts

@@ -175,7 +175,7 @@ let destroys: Array<{ appUid: string; ownerId: number | undefined }>;
 
 beforeAll(async () => {
     env = await setupPuterTestEnv({
-        events: { enabled: true },
+        events: { enabled: true, userHandlers: true },
         unlimitedMetering: true,
     } as IConfig);
 
@@ -204,6 +204,7 @@ afterAll(async () => {
 
 beforeEach(async () => {
     await env.server.clients.db.write('DELETE FROM `event_handlers`', []);
+    await env.server.clients.db.write('DELETE FROM `event_user_handlers`', []);
     await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []);
     await env.server.clients.db.write('DELETE FROM `apps`', []);
     creates.length = 0;
@@ -539,6 +540,8 @@ describe('POST /events/workers/destroy', () => {
 
         const suspended = await env.server.stores.durableSubscription.listByHandler(
             app.uid,
+            'app',
+            0,
             'ingestUpload',
             { suspendedReason: 'handler_not_found' },
         );
@@ -554,6 +557,8 @@ describe('POST /events/workers/destroy', () => {
         const stillSuspended =
             await env.server.stores.durableSubscription.listByHandler(
                 app.uid,
+                'app',
+                0,
                 'ingestUpload',
                 { suspendedReason: 'handler_not_found' },
             );
@@ -621,3 +626,319 @@ describe('POST /events/workers/destroy', () => {
         expect(destroys).toEqual([]);
     });
 });
+
+// -- User scopes -----------------------------------------------------
+
+/** `userId`'s own session of `app`, wherever the app's owner is. */
+const sessionIn = async (app: { uid: string }, asUser: 'user' | 'other') => {
+    const { actor } = await env.server.services.auth.authenticate(
+        asUser === 'user' ? env.users.user.token : env.users.other.token,
+    );
+    return env.server.services.auth.getUserAppToken(actor!, app.uid);
+};
+
+const publishUserScope = (token: string, name: string, source: string) =>
+    call('POST', '/events/handlers/publish', token, {
+        scope: 'user',
+        name,
+        source,
+    });
+
+describe('GET /events/workers with user scopes', () => {
+    it('adds the caller`s user scopes to their app workers outside an app', async () => {
+        const mine = await makeApp(userId);
+        await publish(mine.token, {
+            appUid: mine.uid,
+            name: 'a',
+            source: SOURCE,
+        });
+        const theirs = await makeApp(otherUserId);
+        const inTheirs = await sessionIn(theirs, 'user');
+        await publishUserScope(inTheirs, 'x', SOURCE);
+        await publishUserScope(inTheirs, 'y', OTHER_SOURCE);
+        // Another holder's scope in the same app is not the caller's.
+        await publishUserScope(await sessionIn(theirs, 'other'), 'z', SOURCE);
+
+        const listed = await call(
+            'GET',
+            '/events/workers',
+            env.users.user.token,
+        );
+
+        expect(listed.status).toBe(200);
+        expect(listed.body.items).toEqual([
+            expect.objectContaining({
+                scope: 'app',
+                appUid: mine.uid,
+                handlerCount: 1,
+            }),
+            expect.objectContaining({
+                scope: 'user',
+                appUid: theirs.uid,
+                appName: theirs.uid,
+                handlerCount: 2,
+            }),
+        ]);
+        expect(listed.body.cursor).toBeUndefined();
+    });
+
+    it('pages through the app workers and then the user scopes', async () => {
+        const mine = await makeApp(userId);
+        await publish(mine.token, {
+            appUid: mine.uid,
+            name: 'a',
+            source: SOURCE,
+        });
+        const first = await makeApp(otherUserId);
+        const second = await makeApp(otherUserId);
+        await publishUserScope(await sessionIn(first, 'user'), 'x', SOURCE);
+        await publishUserScope(await sessionIn(second, 'user'), 'x', SOURCE);
+
+        const seen: Array<Record<string, unknown>> = [];
+        let cursor: string | undefined;
+        let pages = 0;
+        do {
+            const page = await call(
+                'GET',
+                `/events/workers?limit=1${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`,
+                env.users.user.token,
+            );
+            expect(page.status).toBe(200);
+            seen.push(...(page.body.items as Array<Record<string, unknown>>));
+            cursor = page.body.cursor as string | undefined;
+            pages += 1;
+        } while (cursor && pages < 10);
+
+        expect(seen.map((item) => [item.scope, item.appUid])).toEqual([
+            ['app', mine.uid],
+            ...[first.uid, second.uid].sort().map((uid) => ['user', uid]),
+        ]);
+    });
+
+    it('lists only that app`s scope for the caller inside an app', async () => {
+        const theirs = await makeApp(otherUserId);
+        const otherApp = await makeApp(otherUserId);
+        const inTheirs = await sessionIn(theirs, 'user');
+        await publishUserScope(inTheirs, 'x', SOURCE);
+        await publishUserScope(await sessionIn(otherApp, 'user'), 'x', SOURCE);
+        await publishUserScope(await sessionIn(theirs, 'other'), 'x', SOURCE);
+        // The owner's own worker in that app is not the caller's to see.
+        await publish(theirs.token, {
+            appUid: theirs.uid,
+            name: 'owner',
+            source: SOURCE,
+        });
+
+        const listed = await call('GET', '/events/workers', inTheirs);
+
+        expect(listed.status).toBe(200);
+        expect(listed.body.items).toEqual([
+            expect.objectContaining({
+                scope: 'user',
+                appUid: theirs.uid,
+                handlerCount: 1,
+            }),
+        ]);
+    });
+
+    it('still lists the owner`s worker beside their own user scope inside the app', async () => {
+        const mine = await makeApp(userId);
+        await publish(mine.token, {
+            appUid: mine.uid,
+            name: 'a',
+            source: SOURCE,
+        });
+        await publishUserScope(mine.token, 'x', SOURCE);
+
+        const listed = await call('GET', '/events/workers', mine.token);
+
+        expect(
+            (listed.body.items as Array<Record<string, unknown>>).map(
+                (item) => item.scope,
+            ),
+        ).toEqual(['app', 'user']);
+    });
+
+    it('refuses a scoped token, which has no scope of its own', async () => {
+        const theirs = await makeApp(otherUserId);
+        await publishUserScope(await sessionIn(theirs, 'user'), 'x', SOURCE);
+        const scoped = await appIssuedScopedToken({
+            uid: theirs.uid,
+            token: await sessionIn(theirs, 'user'),
+        });
+
+        const listed = await call('GET', '/events/workers', scoped);
+        expect(listed.status).toBe(403);
+        expect(listed.body.code).toBe('events_worker_owner_only');
+    });
+});
+
+describe('POST /events/workers/destroy with a user scope', () => {
+    it('removes the caller`s scope only, suspends what was bound, and announces it', async () => {
+        const theirs = await makeApp(otherUserId);
+        const mineIn = await sessionIn(theirs, 'user');
+        const otherIn = await sessionIn(theirs, 'other');
+        await publish(theirs.token, {
+            appUid: theirs.uid,
+            name: 'owner',
+            source: SOURCE,
+        });
+        await publishUserScope(mineIn, 'x', SOURCE);
+        await publishUserScope(mineIn, 'y', OTHER_SOURCE);
+        await publishUserScope(otherIn, 'x', SOURCE);
+        destroys.length = 0;
+
+        const destroyed = await call(
+            'POST',
+            '/events/workers/destroy',
+            mineIn,
+            {
+                scope: 'user',
+            },
+        );
+
+        expect(destroyed.status).toBe(200);
+        expect(destroyed.body).toMatchObject({
+            appUid: theirs.uid,
+            removed: 2,
+            suspended: 0,
+            scope: 'user',
+        });
+        const store = env.server.stores.eventHandler;
+        expect(
+            await store.countForApp({ appUid: theirs.uid, holder: userId }),
+        ).toBe(0);
+        expect(
+            await store.countForApp({
+                appUid: theirs.uid,
+                holder: otherUserId,
+            }),
+        ).toBe(1);
+        expect(await store.countForApp({ appUid: theirs.uid, holder: 0 })).toBe(
+            1,
+        );
+        expect(destroys).toEqual([{ appUid: theirs.uid, ownerId: userId }]);
+
+        const again = await call('POST', '/events/workers/destroy', mineIn, {
+            scope: 'user',
+        });
+        expect(again.status).toBe(404);
+    });
+
+    it('refuses a scoped token and an app session naming another app', async () => {
+        const theirs = await makeApp(otherUserId);
+        const mineIn = await sessionIn(theirs, 'user');
+        await publishUserScope(mineIn, 'x', SOURCE);
+        const scoped = await appIssuedScopedToken({
+            uid: theirs.uid,
+            token: mineIn,
+        });
+        const elsewhere = await makeApp(otherUserId);
+
+        for (const [token, body] of [
+            [scoped, { scope: 'user' }],
+            [mineIn, { scope: 'user', appUid: elsewhere.uid }],
+        ] as const) {
+            const refused = await call(
+                'POST',
+                '/events/workers/destroy',
+                token,
+                body,
+            );
+            expect(refused.status).toBe(403);
+            expect(refused.body.code).toBe('events_handler_forbidden');
+        }
+        expect(
+            await env.server.stores.eventHandler.countForApp({
+                appUid: theirs.uid,
+                holder: userId,
+            }),
+        ).toBe(1);
+    });
+});
+
+describe('destroying a user scope from an account session', () => {
+    it('removes the caller`s own scope of any app, and nobody else`s', async () => {
+        const theirs = await makeApp(otherUserId);
+        await publishUserScope(await sessionIn(theirs, 'user'), 'x', SOURCE);
+        await publishUserScope(await sessionIn(theirs, 'other'), 'x', SOURCE);
+        const store = env.server.stores.eventHandler;
+        destroys.length = 0;
+
+        const destroyed = await call(
+            'POST',
+            '/events/workers/destroy',
+            env.users.user.token,
+            { scope: 'user', appUid: theirs.uid },
+        );
+
+        expect(destroyed.status).toBe(200);
+        expect(destroyed.body).toMatchObject({ removed: 1, scope: 'user' });
+        expect(
+            await store.countForApp({ appUid: theirs.uid, holder: userId }),
+        ).toBe(0);
+        expect(
+            await store.countForApp({ appUid: theirs.uid, holder: otherUserId }),
+        ).toBe(1);
+        expect(destroys).toEqual([{ appUid: theirs.uid, ownerId: userId }]);
+
+        // The same call as the other user removes only theirs; the first user
+        // has no scope left to touch.
+        const none = await call(
+            'POST',
+            '/events/workers/destroy',
+            env.users.user.token,
+            { scope: 'user', appUid: theirs.uid },
+        );
+        expect(none.status).toBe(404);
+        expect(
+            await store.countForApp({ appUid: theirs.uid, holder: otherUserId }),
+        ).toBe(1);
+    });
+
+    it('needs an appUid, and still refuses a scoped token', async () => {
+        const theirs = await makeApp(otherUserId);
+        const mineIn = await sessionIn(theirs, 'user');
+        await publishUserScope(mineIn, 'x', SOURCE);
+
+        const noApp = await call(
+            'POST',
+            '/events/workers/destroy',
+            env.users.user.token,
+            { scope: 'user' },
+        );
+        expect(noApp.status).toBe(403);
+
+        const scoped = await appIssuedScopedToken({
+            uid: theirs.uid,
+            token: mineIn,
+        });
+        const refused = await call('POST', '/events/workers/destroy', scoped, {
+            scope: 'user',
+            appUid: theirs.uid,
+        });
+        expect(refused.status).toBe(403);
+    });
+
+    it('works with user handlers switched off, so a user can stop paying', async () => {
+        const theirs = await makeApp(otherUserId);
+        await publishUserScope(await sessionIn(theirs, 'user'), 'x', SOURCE);
+        const config = (
+            env.server.services.events as unknown as {
+                config: { events: { userHandlers?: boolean } };
+            }
+        ).config;
+        config.events.userHandlers = false;
+        try {
+            const destroyed = await call(
+                'POST',
+                '/events/workers/destroy',
+                env.users.user.token,
+                { scope: 'user', appUid: theirs.uid },
+            );
+            expect(destroyed.status).toBe(200);
+        } finally {
+            config.events.userHandlers = true;
+        }
+    });
+});

src/backend/services/index.ts

@@ -36,6 +36,7 @@ import { FSService } from './fs/FSService';
 import { KvShareService } from './kvShare/KvShareService';
 import { ServerHealthService } from './health/ServerHealthService';
 import { PuterHomepageService } from './homepage/PuterHomepageService';
+import { JobScheduler } from './jobs/JobScheduler';
 import { LocalWorkerService } from './localworker/LocalWorkerService';
 import { MeteringService } from './metering/MeteringService';
 import { NotificationService } from './notification/NotificationService';
@@ -60,6 +61,7 @@ import { UserAccountService } from './user/UserAccountService';
  */
 declare module './types' {
     interface IPuterServiceInstances {
+        jobs: JobScheduler;
         metering: MeteringService;
         appOriginBlocklist: AppOriginBlocklistService;
         permission: PermissionService;
@@ -103,6 +105,9 @@ declare module './types' {
 // NotificationService depends on notification store (for DB) + event client (for socket push).
 // BroadcastService is independent — only needs the event client.
 export const puterServices = {
+    // First, so every later service can register jobs with it. Reaches
+    // `broadcast` at call time only.
+    jobs: JobScheduler,
     metering: MeteringService,
     // Declared before `auth` so AuthService sees it as a prior peer — it
     // queries the blocklist on app-token acquisition and per-request app

src/backend/services/jobs/JobScheduler.test.ts

@@ -0,0 +1,781 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { randomUUID } from 'node:crypto';
+import MockRedis from 'ioredis-mock';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import {
+    JobLeaseLostError,
+    JobScheduler,
+    type JobRunContext,
+} from './JobScheduler.js';
+
+type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
+
+const HOUR = 60 * 60_000;
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+describe('JobScheduler', () => {
+    let server: PuterServer;
+    const schedulers: JobScheduler[] = [];
+
+    beforeAll(async () => {
+        server = await setupTestServer();
+    });
+
+    afterAll(async () => {
+        await server?.shutdown();
+    });
+
+    afterEach(() => {
+        vi.restoreAllMocks();
+        vi.useRealTimers();
+        for (const scheduler of schedulers.splice(0)) {
+            scheduler.onServerPrepareShutdown();
+        }
+    });
+
+    const db = () => server.clients.db;
+
+    /**
+     * A node of its own: its own config and region, on the test server's Redis
+     * unless given one.
+     */
+    const node = ({
+        namespace,
+        region = 'r1',
+        redis,
+        domain = 'puter.test',
+        enabled,
+    }: {
+        namespace?: string;
+        region?: string;
+        redis?: InstanceType<typeof MockRedis>;
+        domain?: string;
+        enabled?: boolean;
+    } = {}) => {
+        const config = {
+            port: 0,
+            extensions: [],
+            domain,
+            serverId: `node-${schedulers.length}`,
+            jobs: { namespace, enabled },
+        } as IConfig;
+        const clients = (
+            redis ? { ...server.clients, redis } : server.clients
+        ) as SchedulerArgs[1];
+        const services = {
+            broadcast: { regionId: region },
+        } as unknown as SchedulerArgs[3];
+        const scheduler = new JobScheduler(
+            config,
+            clients,
+            server.stores as SchedulerArgs[2],
+            services,
+        );
+        schedulers.push(scheduler);
+        return scheduler;
+    };
+
+    const freshNamespace = () => `ns-${randomUUID()}`;
+
+    /**
+     * A job that runs `batches` batches `batchMs` apart, checking its signal
+     * before each.
+     */
+    const batchedJob = (batches: number, batchMs: number) => {
+        const seen = { done: 0, signal: null as AbortSignal | null };
+        const run = async ({ signal }: JobRunContext) => {
+            seen.signal = signal;
+            for (let i = 0; i < batches; i++) {
+                if (signal.aborted) return;
+                await sleep(batchMs);
+                seen.done += 1;
+            }
+        };
+        return { seen, run };
+    };
+
+    const keysOf = (namespace: string, name: string) => ({
+        lock: `jobs:{${namespace}:${name}}:lock`,
+        gate: `jobs:{${namespace}:${name}}:next`,
+        run: `jobs:{${namespace}:${name}}:run`,
+    });
+
+    // -- Registration -------------------------------------------------
+
+    it('refuses a bad or duplicate registration', () => {
+        const scheduler = node({ namespace: freshNamespace() });
+        const run = async () => {};
+        expect(() =>
+            scheduler.register('bad name', {
+                every: 1000,
+                scope: 'fleet',
+                run,
+            }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('job', { every: 0, scope: 'fleet', run }),
+        ).toThrow(/every/);
+        expect(() =>
+            scheduler.register('job', {
+                every: 1000,
+                scope: 'fleet',
+                leaseMs: 1.5,
+                run,
+            }),
+        ).toThrow(/leaseMs/);
+        // `:` would let two (namespace, name) pairs share Redis keys.
+        expect(() =>
+            scheduler.register('b:c', { every: 1000, scope: 'fleet', run }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('x'.repeat(129), {
+                every: 1000,
+                scope: 'fleet',
+                run,
+            }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('job.region', {
+                every: 1000,
+                scope: 'region',
+                leaseMs: 2000,
+                run,
+            }),
+        ).toThrow(/exceeds every/);
+        scheduler.register('job', { every: 1000, scope: 'fleet', run });
+        expect(() =>
+            scheduler.register('job', { every: 1000, scope: 'fleet', run }),
+        ).toThrow(/already registered/);
+    });
+
+    it('falls back to the default namespace, once and loudly, when the configured one cannot be held', () => {
+        const run = async () => {};
+        const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+        for (const namespace of ['a:b', 'x'.repeat(129), 'spaced name']) {
+            const scheduler = node({ namespace });
+            scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
+            expect(scheduler.namespace).toBe('default');
+        }
+        logged.mockClear();
+
+        // A self-host whose domain carries a port still boots.
+        const scheduler = node({ domain: 'localhost:4100' });
+        scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
+        scheduler.register('job.b', { every: 1000, scope: 'fleet', run });
+        expect(scheduler.namespace).toBe('default');
+        expect(logged).toHaveBeenCalledTimes(1);
+        expect(String(logged.mock.calls[0][0])).toMatch(
+            /localhost:4100.*jobs\.namespace/,
+        );
+    });
+
+    it('does nothing at all while no job is registered', async () => {
+        vi.useFakeTimers({
+            toFake: [
+                'setInterval',
+                'setTimeout',
+                'clearInterval',
+                'clearTimeout',
+            ],
+        });
+        // Timers the test runtime itself already holds.
+        const baseline = vi.getTimerCount();
+        const reads = vi.spyOn(db(), 'read');
+        const preads = vi.spyOn(db(), 'pread');
+        const writes = vi.spyOn(db(), 'write');
+        const scheduler = node({ namespace: freshNamespace() });
+
+        scheduler.onServerStart();
+        expect(vi.getTimerCount()).toBe(baseline);
+        await scheduler.poll();
+        expect(reads).not.toHaveBeenCalled();
+        expect(preads).not.toHaveBeenCalled();
+        expect(writes).not.toHaveBeenCalled();
+
+        // A job registered after start arms the poll; shutdown disarms it.
+        scheduler.register('late', {
+            every: HOUR,
+            scope: 'fleet',
+            run: async () => {},
+        });
+        expect(vi.getTimerCount()).toBeGreaterThan(baseline);
+        scheduler.onServerPrepareShutdown();
+        expect(vi.getTimerCount()).toBe(baseline);
+    });
+
+    it('claims nothing once draining, and leaves a running job its lease', async () => {
+        const scheduler = node({ namespace: `drain${Date.now()}` });
+        let release!: () => void;
+        let signal!: AbortSignal;
+        scheduler.register('long', {
+            every: HOUR,
+            scope: 'region',
+            run: (ctx) => {
+                signal = ctx.signal;
+                return new Promise<void>((resolve) => {
+                    release = resolve;
+                });
+            },
+        });
+        const first = scheduler.poll();
+        await vi.waitFor(() => expect(release).toBeDefined());
+        const run = vi.fn(async () => {});
+        scheduler.register('other', { every: HOUR, scope: 'region', run });
+
+        scheduler.onServerDrainStart();
+        await scheduler.poll();
+        expect(run).not.toHaveBeenCalled();
+        expect(signal.aborted).toBe(false);
+
+        release();
+        await first;
+    });
+
+    it('arms no poll on a node with jobs turned off, though poll() still runs them', async () => {
+        vi.useFakeTimers({
+            toFake: [
+                'setInterval',
+                'setTimeout',
+                'clearInterval',
+                'clearTimeout',
+            ],
+        });
+        const baseline = vi.getTimerCount();
+        const scheduler = node({ namespace: freshNamespace(), enabled: false });
+        const warned = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let runs = 0;
+        scheduler.onServerStart();
+        scheduler.register('job.off', {
+            every: HOUR,
+            scope: 'fleet',
+            run: async () => {
+                runs += 1;
+            },
+        });
+        scheduler.register('job.off.too', {
+            every: HOUR,
+            scope: 'region',
+            run: async () => undefined,
+        });
+        expect(vi.getTimerCount()).toBe(baseline);
+        // Said once per node, not per job.
+        expect(
+            warned.mock.calls.filter(([line]) =>
+                String(line).includes('jobs.enabled'),
+            ),
+        ).toHaveLength(1);
+        vi.useRealTimers();
+
+        await scheduler.poll();
+        expect(runs).toBe(1);
+    });
+
+    describe('on a controlled clock', () => {
+        const leaseMs = 300;
+        /**
+         * Pending timeouts armed from JobScheduler.ts. The shared test server's
+         * own loops keep running on real intervals and may arm fake timers
+         * mid-test, so the global timer count says nothing about the
+         * scheduler.
+         */
+        let armed: Set<unknown>;
+        const schedulerTimers = () => armed.size;
+
+        beforeEach(() => {
+            vi.useFakeTimers({
+                toFake: [
+                    'setTimeout',
+                    'clearTimeout',
+                    'setInterval',
+                    'clearInterval',
+                    'performance',
+                ],
+            });
+            armed = new Set();
+            const fakeSetTimeout = globalThis.setTimeout;
+            const fakeClearTimeout = globalThis.clearTimeout;
+            const fromScheduler = () => {
+                const limit = Error.stackTraceLimit;
+                Error.stackTraceLimit = 30;
+                const stack = new Error().stack ?? '';
+                Error.stackTraceLimit = limit;
+                return /services[\\/]jobs[\\/]JobScheduler\.ts/.test(stack);
+            };
+            vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+                fn: (...args: unknown[]) => void,
+                ms?: number,
+                ...rest: unknown[]
+            ) => {
+                if (!fromScheduler()) return fakeSetTimeout(fn, ms, ...rest);
+                const handle = fakeSetTimeout(
+                    (...args: unknown[]) => {
+                        armed.delete(handle);
+                        fn(...args);
+                    },
+                    ms,
+                    ...rest,
+                );
+                armed.add(handle);
+                return handle;
+            }) as never);
+            vi.spyOn(globalThis, 'clearTimeout').mockImplementation(((
+                handle: Parameters<typeof clearTimeout>[0],
+            ) => {
+                armed.delete(handle);
+                fakeClearTimeout(handle);
+            }) as never);
+        });
+
+        /** Batches of `batchMs`, each started only while the signal is live. */
+        const timedJob = (batches: number, batchMs: number) => {
+            const seen = {
+                starts: [] as number[],
+                abortedAt: null as number | null,
+                signal: null as AbortSignal | null,
+                saveError: null as unknown,
+            };
+            const run = async ({ signal, saveState }: JobRunContext) => {
+                seen.signal = signal;
+                signal.addEventListener('abort', () => {
+                    seen.abortedAt = performance.now();
+                });
+                for (let i = 0; i < batches; i++) {
+                    if (signal.aborted) break;
+                    seen.starts.push(performance.now());
+                    await sleep(batchMs);
+                }
+                try {
+                    await saveState({ late: true });
+                } catch (err) {
+                    seen.saveError = err;
+                }
+            };
+            return { seen, run };
+        };
+
+        /** Let the run play out, then check nothing it armed is left behind. */
+        const settle = async (running: Promise<void>) => {
+            await vi.advanceTimersByTimeAsync(2_000);
+            await running;
+            expect(schedulerTimers()).toBe(0);
+        };
+
+        it('clears a run’s timers at shutdown even while the run hangs', async () => {
+            const scheduler = node({ namespace: freshNamespace() });
+            let release!: () => void;
+            scheduler.register('job.hang', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run: () =>
+                    new Promise<void>((resolve) => {
+                        release = resolve;
+                    }),
+            });
+            const running = scheduler.poll();
+            await vi.advanceTimersByTimeAsync(10);
+            expect(schedulerTimers()).toBeGreaterThan(0);
+
+            scheduler.onServerPrepareShutdown();
+            expect(schedulerTimers()).toBe(0);
+            release();
+            await settle(running);
+        });
+
+        it('never starts a region run once taking the lock used up half the lease', async () => {
+            const namespace = freshNamespace();
+            const redis = new MockRedis({ port: 17104 });
+            const scheduler = node({ namespace, redis });
+            const run = vi.fn(async () => {});
+            scheduler.register('job.slowLock', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run,
+            });
+            const set = redis.set.bind(redis) as (
+                ...args: unknown[]
+            ) => Promise<unknown>;
+            const slowLock = vi.spyOn(redis, 'set').mockImplementation((async (
+                ...args: unknown[]
+            ) => {
+                const result = await set(...args);
+                if (String(args[0]).endsWith(':lock')) {
+                    await sleep(leaseMs / 2 + 10);
+                }
+                return result;
+            }) as never);
+            await settle(scheduler.poll());
+
+            expect(run).not.toHaveBeenCalled();
+            // The gate went back, so the next poll runs it.
+            expect(
+                await redis.exists(`jobs:{${namespace}:job.slowLock}:next`),
+            ).toBe(0);
+            slowLock.mockRestore();
+            await settle(scheduler.poll());
+            expect(run).toHaveBeenCalledTimes(1);
+        });
+
+        it('aborts a region run before its next batch once the lock lapses, and leaves it due at once', async () => {
+            const namespace = freshNamespace();
+            const redis = new MockRedis({ port: 17106 });
+            const scheduler = node({ namespace, redis });
+            const job = timedJob(10, 70);
+            scheduler.register('job.lapse', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run: job.run,
+            });
+            const lockedAt = performance.now();
+            await settle(scheduler.poll());
+
+            expect(job.seen.abortedAt! - lockedAt).toBe(leaseMs);
+            expect((job.seen.signal?.reason as Error).message).toMatch(
+                /lapsed/,
+            );
+            expect(job.seen.starts.length).toBeLessThan(10);
+            expect(job.seen.starts.every((t) => t < job.seen.abortedAt!)).toBe(
+                true,
+            );
+            // The gate is still this run's, so its last save went through;
+            // interrupted, the gate goes back and the job is due at once.
+            expect(job.seen.saveError).toBeNull();
+            const keys = {
+                gate: `jobs:{${namespace}:job.lapse}:next`,
+                run: `jobs:{${namespace}:job.lapse}:run`,
+            };
+            expect(await redis.exists(keys.gate)).toBe(0);
+            expect(await redis.hget(keys.run, 'state')).toBe('{"late":true}');
+            expect(await redis.hget(keys.run, 'failures')).toBeNull();
+        });
+    });
+
+    it('refuses state over 64 KB', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17107 });
+        const scheduler = node({ namespace, redis });
+        let refused: unknown = null;
+        scheduler.register('job.bigState', {
+            every: HOUR,
+            scope: 'region',
+            run: async ({ saveState }) => {
+                try {
+                    await saveState({ blob: 'x'.repeat(70_000) });
+                } catch (err) {
+                    refused = err;
+                }
+            },
+        });
+        await scheduler.poll();
+        expect((refused as Error).message).toMatch(/over 65536 bytes/);
+        expect(
+            await redis.hget(keysOf(namespace, 'job.bigState').run, 'state'),
+        ).toBeNull();
+    });
+
+    it('releases an interrupted run on shutdown without counting a failure', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17109 });
+        const scheduler = node({ namespace, redis });
+        const job = batchedJob(50, 50);
+        scheduler.register('job.shutdown', {
+            every: HOUR,
+            scope: 'region',
+            run: job.run,
+        });
+        const running = scheduler.poll();
+        await vi.waitFor(() => expect(job.seen.done).toBeGreaterThan(0));
+        scheduler.onServerPrepareShutdown();
+        await running;
+
+        expect(job.seen.done).toBeLessThan(50);
+        expect((job.seen.signal?.reason as Error).message).toBe(
+            'server shutting down',
+        );
+        const keys = keysOf(namespace, 'job.shutdown');
+        // Due again at once, with no failure counted.
+        expect(await redis.exists(keys.gate)).toBe(0);
+        expect(await redis.hget(keys.run, 'failures')).toBeNull();
+    });
+
+    it('keeps namespaces apart: the same job name runs once in each', async () => {
+        const redis = new MockRedis({ port: 17110 });
+        const [nsA, nsB] = [freshNamespace(), freshNamespace()];
+        const nodes = [
+            node({ namespace: nsA, redis }),
+            node({ namespace: nsA, redis }),
+            node({ namespace: nsB, redis }),
+            node({ namespace: nsB, redis }),
+        ];
+        const runs: string[] = [];
+        nodes.forEach((scheduler, i) =>
+            scheduler.register('job.shared', {
+                every: HOUR,
+                scope: 'region',
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    await sleep(20);
+                },
+            }),
+        );
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+    });
+
+    it('falls back to the domain as its namespace', () => {
+        expect(node({ domain: 'example.test' }).namespace).toBe('example.test');
+        expect(
+            node({ namespace: 'explicit', domain: 'example.test' }).namespace,
+        ).toBe('explicit');
+    });
+
+    it('boots a server whose domain carries a port, registering its jobs under the default namespace', async () => {
+        const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+        const portDomain = await setupTestServer({
+            domain: 'localhost:4101',
+        } as IConfig);
+        try {
+            expect(portDomain.services.jobs.namespace).toBe('default');
+            const namespaceErrors = logged.mock.calls.filter(([message]) =>
+                String(message).startsWith('[jobs] namespace'),
+            );
+            expect(namespaceErrors).toHaveLength(1);
+        } finally {
+            await portDomain.shutdown();
+        }
+    });
+
+    // -- Region scope -------------------------------------------------
+
+    it('runs a region job once per region', async () => {
+        const namespace = freshNamespace();
+        const redisA = new MockRedis({ port: 17101 });
+        const redisB = new MockRedis({ port: 17102 });
+        const nodes = [
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'b', redis: redisB }),
+            node({ namespace, region: 'b', redis: redisB }),
+        ];
+        const runs: string[] = [];
+        nodes.forEach((scheduler, i) =>
+            scheduler.register('job.region', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs: 5_000,
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    await sleep(20);
+                },
+            }),
+        );
+
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+        expect(
+            await redisA.pttl(`jobs:{${namespace}:job.region}:next`),
+        ).toBeGreaterThan(HOUR - 60_000);
+        // The region lock was released after the run.
+        expect(await redisA.exists(`jobs:{${namespace}:job.region}:lock`)).toBe(
+            0,
+        );
+    });
+
+    it('backs off a failing region job and keeps its state in Redis', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17103 });
+        const scheduler = node({ namespace, redis });
+        const seen: unknown[] = [];
+        let fail = true;
+        scheduler.register('job.regionFail', {
+            every: HOUR,
+            scope: 'region',
+            leaseMs: 5_000,
+            run: async ({ state, saveState }) => {
+                seen.push(state);
+                await saveState({ page: seen.length });
+                if (fail) throw new Error('nope');
+            },
+        });
+        const gate = `jobs:{${namespace}:job.regionFail}:next`;
+
+        await scheduler.poll();
+        expect(
+            await redis.hget(
+                `jobs:{${namespace}:job.regionFail}:run`,
+                'failures',
+            ),
+        ).toBe('1');
+        expect(await redis.pttl(gate)).toBeLessThanOrEqual(2_000);
+
+        fail = false;
+        await redis.del(gate);
+        await scheduler.poll();
+        expect(seen).toEqual([null, { page: 1 }]);
+        expect(
+            await redis.hget(
+                `jobs:{${namespace}:job.regionFail}:run`,
+                'failures',
+            ),
+        ).toBeNull();
+        expect(await redis.pttl(gate)).toBeGreaterThan(HOUR - 60_000);
+    });
+
+    it('skips region jobs quietly while Redis is unreachable', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17108 });
+        const scheduler = node({ namespace, redis });
+        const run = vi.fn(async () => {});
+        scheduler.register('job.noRedis', {
+            every: HOUR,
+            scope: 'region',
+            run,
+        });
+        vi.spyOn(redis, 'exists').mockRejectedValue(
+            new Error('connection refused'),
+        );
+        const errors = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+        await scheduler.poll();
+        expect(run).not.toHaveBeenCalled();
+        expect(errors).not.toHaveBeenCalled();
+    });
+
+    it('fences a region run that outlived its gate off its successor’s keys', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17105 });
+        const [slow, next] = [
+            node({ namespace, redis }),
+            node({ namespace, redis }),
+        ];
+        const keys = {
+            gate: `jobs:{${namespace}:job.fenced}:next`,
+            run: `jobs:{${namespace}:job.fenced}:run`,
+        };
+        let release!: () => void;
+        let slowStarted = false;
+        let slowSaveError: unknown = null;
+        // Ignores its signal and outlives both its 200 ms lock and 400 ms gate.
+        slow.register('job.fenced', {
+            every: 400,
+            scope: 'region',
+            leaseMs: 200,
+            run: async ({ saveState }) => {
+                slowStarted = true;
+                await new Promise<void>((resolve) => {
+                    release = resolve;
+                });
+                try {
+                    await saveState({ by: 'slow' });
+                } catch (err) {
+                    slowSaveError = err;
+                }
+                throw new Error('slow run failed');
+            },
+        });
+        next.register('job.fenced', {
+            every: 400,
+            scope: 'region',
+            leaseMs: 200,
+            run: async ({ saveState }) => {
+                await saveState({ by: 'next' });
+            },
+        });
+
+        const slowRun = slow.poll();
+        await vi.waitFor(() => expect(slowStarted).toBe(true));
+        await sleep(450);
+        await next.poll();
+        const successorGate = await redis.get(keys.gate);
+        expect(successorGate).not.toBeNull();
+
+        release();
+        await slowRun;
+        expect(slowSaveError).toBeInstanceOf(JobLeaseLostError);
+        // Neither the slow run's state, its failure count, nor its cleanup
+        // touched what the successor holds.
+        expect(await redis.get(keys.gate)).toBe(successorGate);
+        expect(await redis.pttl(keys.gate)).toBeGreaterThan(0);
+        expect(await redis.hget(keys.run, 'state')).toBe('{"by":"next"}');
+        expect(await redis.hget(keys.run, 'failures')).toBeNull();
+    });
+    it('runs a fleet job once per region, its lease capped at its cadence and nearPrimary ignored', async () => {
+        const namespace = freshNamespace();
+        const redisA = new MockRedis({ port: 17111 });
+        const redisB = new MockRedis({ port: 17112 });
+        const nodes = [
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'b', redis: redisB }),
+            node({ namespace, region: 'b', redis: redisB }),
+        ];
+        const keys = keysOf(namespace, 'job.fleet');
+        const runs: string[] = [];
+        const lockTtls: number[] = [];
+        const reads = vi.spyOn(db(), 'read');
+        const preads = vi.spyOn(db(), 'pread');
+        const writes = vi.spyOn(db(), 'write');
+        nodes.forEach((scheduler, i) => {
+            const redis = i < 2 ? redisA : redisB;
+            scheduler.register('job.fleet', {
+                every: 2_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                // Longer than `every`: refused for a region job, capped here.
+                leaseMs: HOUR,
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    lockTtls.push(await redis.pttl(keys.lock));
+                    await sleep(20);
+                },
+            });
+        });
+
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+        for (const ttl of lockTtls) {
+            expect(ttl).toBeGreaterThan(0);
+            expect(ttl).toBeLessThanOrEqual(2_000);
+        }
+        expect(await redisA.pttl(keys.gate)).toBeGreaterThan(1_000);
+        expect(await redisB.pttl(keys.gate)).toBeGreaterThan(1_000);
+        expect(reads).not.toHaveBeenCalled();
+        expect(preads).not.toHaveBeenCalled();
+        expect(writes).not.toHaveBeenCalled();
+    });
+});

src/backend/services/jobs/JobScheduler.ts

@@ -0,0 +1,528 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { withRedisLock } from '../../util/redisLock.js';
+import { resolveJobsNamespace } from '../../util/jobsNamespace.js';
+import { deliveryBackoffMs } from '../../util/retryBackoff.js';
+import { PuterService } from '../types';
+import {
+    JobLeaseLostError,
+    asError,
+    lostLease,
+    parseState,
+    runOutcome,
+    serializeState,
+    type RegisteredJob,
+    type RunLease,
+} from './jobRun.js';
+
+export { JobLeaseLostError } from './jobRun.js';
+
+/** How often each node looks for due jobs. */
+const JOB_POLL_MS = 30_000;
+/** Each poll waits up to this long first, so nodes don't all claim at once. */
+const JOB_POLL_JITTER_MS = 5_000;
+const JOB_DEFAULT_LEASE_MS = 15 * 60_000;
+/** Consecutive failures that raise `scheduled_job_failing`. */
+const JOB_FAILING_ALARM_AT = 3;
+const JOB_SHUTDOWN_WAIT_MS = 10_000;
+/**
+ * Job names: ASCII, at most 128, and without `:`, so a (namespace, name) pair
+ * maps to exactly one set of Redis keys (namespaces follow the same rule).
+ */
+const JOB_ID_PATTERN = /^[A-Za-z0-9._-]{1,128}$/;
+
+// Region writes apply only while the gate still holds this run's token.
+const REGION_FENCED_WRITE = `
+if redis.call('get', KEYS[1]) ~= ARGV[1] then return -1 end
+local op = ARGV[2]
+if op == 'hset' then return redis.call('hset', KEYS[2], ARGV[3], ARGV[4]) end
+if op == 'hdel' then return redis.call('hdel', KEYS[2], ARGV[3]) end
+if op == 'hincrby' then return redis.call('hincrby', KEYS[2], ARGV[3], 1) end
+if op == 'pexpire' then return redis.call('pexpire', KEYS[1], ARGV[3]) end
+if op == 'del' then return redis.call('del', KEYS[1]) end
+return -1
+`;
+
+/**
+ * `region` runs once per cadence per region. `fleet` means once across every
+ * region where `services.jobs` supports it; this default runs it per region.
+ */
+export type JobScope = 'fleet' | 'region';
+
+export interface JobRunContext {
+    /** Aborts once the run may have lost the job. Check it before each batch. */
+    signal: AbortSignal;
+    /** What an earlier run saved, or null. */
+    state: unknown;
+    /**
+     * Persist JSON-serializable `state` for later runs. Throws once the run
+     * lost the job.
+     */
+    saveState: (state: unknown) => Promise<void>;
+}
+
+export interface JobDefinition {
+    /** Cadence in ms. */
+    every: number;
+    scope: JobScope;
+    /**
+     * Prefer the region nearest the primary database, for a `fleet` job. The
+     * default scheduler ignores it.
+     */
+    nearPrimary?: boolean;
+    /**
+     * How long a run holds the job, in total on the region path (at most
+     * `every`). Default 15 min, capped at `every` for `region`; a `fleet` job
+     * run per region is capped at `every` too.
+     */
+    leaseMs?: number;
+    run: (ctx: JobRunContext) => Promise<void>;
+}
+
+/** The `afterId` cursor a keyset-walking job saved in its state, or 0. */
+export const savedAfterId = (state: unknown): number => {
+    const afterId = (state as { afterId?: unknown } | null)?.afterId;
+    return typeof afterId === 'number' &&
+        Number.isSafeInteger(afterId) &&
+        afterId > 0
+        ? afterId
+        : 0;
+};
+
+interface RegionKeys {
+    lock: string;
+    gate: string;
+    run: string;
+}
+
+/**
+ * Runs our own recurring backend work once per cadence per region, under a
+ * Redis lock. `fleet` jobs take the same path here; an extension registered as
+ * `services.jobs` can run them fleet-wide by overriding `pollFleet`. Nothing
+ * polls until a job is registered.
+ */
+export class JobScheduler extends PuterService {
+    #jobs = new Map<string, RegisteredJob>();
+    #running = new Map<string, Promise<void>>();
+    #leases = new Set<RunLease>();
+    #started = false;
+    #stopping = false;
+    /** Polling stopped; runs already going keep their leases. */
+    #draining = false;
+    #timer: ReturnType<typeof setInterval> | null = null;
+    #pollDelays = new Set<ReturnType<typeof setTimeout>>();
+    #warnedOff = false;
+    #namespace: string | null = null;
+
+    /** Keys this deployment's jobs, so deployments sharing Redis stay apart. */
+    get namespace(): string {
+        return (this.#namespace ??= resolveJobsNamespace(this.config));
+    }
+
+    register(name: string, definition: JobDefinition): void {
+        if (!JOB_ID_PATTERN.test(name)) {
+            throw new Error(`invalid job name: ${name}`);
+        }
+        if (this.#jobs.has(name)) {
+            throw new Error(`job already registered: ${name}`);
+        }
+        const { every, scope } = definition;
+        if (!Number.isSafeInteger(every) || every <= 0) {
+            throw new Error(`job ${name}: every must be a positive integer`);
+        }
+        if (scope !== 'fleet' && scope !== 'region') {
+            throw new Error(`job ${name}: unknown scope`);
+        }
+        const leaseMs =
+            definition.leaseMs ??
+            (scope === 'region'
+                ? Math.min(JOB_DEFAULT_LEASE_MS, every)
+                : JOB_DEFAULT_LEASE_MS);
+        if (!Number.isSafeInteger(leaseMs) || leaseMs <= 0) {
+            throw new Error(`job ${name}: leaseMs must be a positive integer`);
+        }
+        // The gate that fences a region run lasts `every`.
+        if (scope === 'region' && leaseMs > every) {
+            throw new Error(
+                `job ${name}: a region job's leaseMs exceeds every`,
+            );
+        }
+        this.#jobs.set(name, {
+            every,
+            scope,
+            nearPrimary: definition.nearPrimary ?? false,
+            leaseMs,
+            run: definition.run,
+        });
+        if (this.#started) this.#arm();
+    }
+
+    override onServerStart(): void {
+        this.#started = true;
+        if (this.#jobs.size > 0) this.#arm();
+    }
+
+    /** Claim nothing new while draining; running jobs carry on until prepare. */
+    override onServerDrainStart(): void {
+        this.#stopPolling();
+    }
+
+    override onServerPrepareShutdown(): void {
+        this.#stopping = true;
+        this.#stopPolling();
+        for (const lease of [...this.#leases]) {
+            lease.controller.abort(
+                new JobLeaseLostError('server shutting down'),
+            );
+            lease.close();
+        }
+    }
+
+    #stopPolling(): void {
+        this.#draining = true;
+        if (this.#timer) clearInterval(this.#timer);
+        this.#timer = null;
+        for (const delay of this.#pollDelays) clearTimeout(delay);
+        this.#pollDelays.clear();
+    }
+
+    /**
+     * Give aborted runs a moment to record themselves before the database
+     * closes.
+     */
+    override async onServerShutdown(): Promise<void> {
+        if (this.#running.size === 0) return;
+        let wait: ReturnType<typeof setTimeout> | undefined;
+        await Promise.race([
+            Promise.allSettled(this.#running.values()),
+            new Promise((resolve) => {
+                wait = setTimeout(resolve, JOB_SHUTDOWN_WAIT_MS);
+            }),
+        ]);
+        clearTimeout(wait);
+    }
+
+    /**
+     * Start every registered job that is due and not already running here.
+     * Resolves once the runs it started have finished.
+     */
+    async poll(): Promise<void> {
+        if (this.#stopping || this.#draining) return;
+        const fleet: string[] = [];
+        const region: Promise<void>[] = [];
+        for (const [name, job] of this.#jobs) {
+            if (this.#running.has(name)) continue;
+            if (job.scope === 'fleet') {
+                fleet.push(name);
+                continue;
+            }
+            region.push(this.track(name, () => this.#tryRegion(name, job)));
+        }
+        await Promise.all([this.pollFleet(fleet), ...region]);
+    }
+
+    #arm(): void {
+        if (this.#timer || this.#stopping || this.#draining) return;
+        if (this.config.jobs?.enabled === false) {
+            // Nothing alarms on a job that never runs, so say it here.
+            if (!this.#warnedOff) {
+                console.warn(
+                    '[jobs] polling is off on this node (jobs.enabled: false); other nodes must run the jobs',
+                );
+            }
+            this.#warnedOff = true;
+            return;
+        }
+        const tick = () => {
+            const delay = setTimeout(() => {
+                this.#pollDelays.delete(delay);
+                void this.#pollSafely();
+            }, Math.random() * JOB_POLL_JITTER_MS);
+            delay.unref?.();
+            this.#pollDelays.add(delay);
+        };
+        const timer = setInterval(tick, JOB_POLL_MS);
+        timer.unref?.();
+        this.#timer = timer;
+        tick();
+    }
+
+    async #pollSafely(): Promise<void> {
+        try {
+            await this.poll();
+        } catch (err) {
+            console.error('[jobs] poll failed', err);
+        }
+    }
+
+    // -- Region scope -------------------------------------------------
+
+    async #tryRegion(name: string, job: RegisteredJob): Promise<void> {
+        const redis = this.clients.redis;
+        const keys = regionKeys(this.namespace, name);
+        let gated: number;
+        try {
+            gated = await redis.exists(keys.gate);
+        } catch {
+            // Redis is unreachable; the lock below would be busy anyway.
+            return;
+        }
+        if (gated > 0) return;
+        // Before the lock, so acquiring it counts against `leaseMs`.
+        const startedAt = performance.now();
+        await withRedisLock(
+            redis,
+            keys.lock,
+            async () => {
+                const token = randomUUID();
+                const gate = await redis.set(
+                    keys.gate,
+                    token,
+                    'PX',
+                    job.every,
+                    'NX',
+                );
+                if (gate !== 'OK') return;
+                await this.#runRegion(name, job, keys, token, startedAt);
+            },
+            {
+                ttlMs: job.leaseMs,
+                onUnavailable: 'busy',
+                onBusy: (): void => undefined,
+            },
+        );
+    }
+
+    /**
+     * One run under the region lock. The lock doesn't renew, so the signal
+     * aborts once it lapses, and every write is fenced on the gate's token.
+     */
+    async #runRegion(
+        name: string,
+        job: RegisteredJob,
+        keys: RegionKeys,
+        token: string,
+        startedAt: number,
+    ): Promise<void> {
+        const lease = this.#regionLease(job.leaseMs, startedAt);
+        const { signal } = lease.controller;
+        const write = (op: string, ...args: (string | number)[]) =>
+            this.#regionWrite(keys, token, op, ...args);
+        let error: unknown = null;
+        try {
+            const saved = await this.clients.redis.hget(keys.run, 'state');
+            if (await lease.ready()) {
+                await job.run({
+                    signal,
+                    state: parseState(saved),
+                    saveState: async (state) => {
+                        const json = serializeState(state);
+                        if ((await write('hset', 'state', json)) < 0) {
+                            lostLease(lease, name);
+                        }
+                    },
+                });
+            }
+        } catch (err) {
+            error = err;
+        } finally {
+            lease.close();
+        }
+
+        const outcome = runOutcome(signal, error);
+        if (outcome === 'success') {
+            await write('hdel', 'failures');
+        } else if (outcome === 'failure') {
+            const failures = await write('hincrby', 'failures');
+            if (failures < 0) return;
+            await write(
+                'pexpire',
+                Math.min(job.every, deliveryBackoffMs(failures)),
+            );
+            this.reportFailure(name, failures, error);
+        } else {
+            await write('del');
+        }
+    }
+
+    /**
+     * The lock lapses `leaseMs` after `from`; the run needs half of it left to
+     * start.
+     */
+    #regionLease(leaseMs: number, from: number): RunLease {
+        const controller = new AbortController();
+        const deadline = from + leaseMs;
+        const abort = (reason: string) => {
+            if (!controller.signal.aborted) {
+                controller.abort(new JobLeaseLostError(reason));
+            }
+        };
+        const lapse = setTimeout(
+            () => abort('region lock lapsed'),
+            Math.max(0, deadline - performance.now()),
+        );
+        const lease: RunLease = {
+            controller,
+            ready: async () => {
+                if (
+                    !controller.signal.aborted &&
+                    deadline - performance.now() >= leaseMs / 2
+                ) {
+                    return true;
+                }
+                abort('too little of the lock left to start');
+                return false;
+            },
+            close: () => {
+                clearTimeout(lapse);
+                this.closeLease(lease);
+            },
+        };
+        this.openLease(lease);
+        return lease;
+    }
+
+    /**
+     * Run a write to the region keys if the gate still holds `token`; -1 if
+     * not.
+     */
+    async #regionWrite(
+        keys: RegionKeys,
+        token: string,
+        op: string,
+        ...args: (string | number)[]
+    ): Promise<number> {
+        const result = await this.clients.redis.eval(
+            REGION_FENCED_WRITE,
+            2,
+            keys.gate,
+            keys.run,
+            token,
+            op,
+            ...args,
+        );
+        return Number(result);
+    }
+
+    // -- Hooks, also for extensions that replace `services.jobs` -------
+
+    /**
+     * Run `fleet` jobs registered here and not running in this process. The
+     * default holds each per region, its lease capped at `every`.
+     */
+    protected async pollFleet(names: string[]): Promise<void> {
+        await Promise.all(
+            names.map((name) => {
+                const job = this.definitionOf(name);
+                return this.track(name, () =>
+                    this.#tryRegion(name, {
+                        ...job,
+                        leaseMs: Math.min(job.leaseMs, job.every),
+                    }),
+                );
+            }),
+        );
+    }
+
+    /** Runs `work` as `name`'s only run in this process. */
+    protected track(name: string, work: () => Promise<void>): Promise<void> {
+        const run = (async () => {
+            try {
+                await work();
+            } catch (err) {
+                console.error(`[jobs] ${name} errored`, err);
+            } finally {
+                this.#running.delete(name);
+            }
+        })();
+        this.#running.set(name, run);
+        return run;
+    }
+
+    protected isRunning(name: string): boolean {
+        return this.#running.has(name);
+    }
+
+    protected definitionOf(name: string): RegisteredJob {
+        const job = this.#jobs.get(name);
+        if (!job) throw new Error(`job not registered: ${name}`);
+        return job;
+    }
+
+    /** Who runs a job here: region, server and process. */
+    protected owner(): string {
+        const region = this.services.broadcast.regionId;
+        const server = this.config.serverId ?? 'local';
+        return `${region}:${server}:${process.pid}`.slice(0, 128);
+    }
+
+    /**
+     * Track `lease` for shutdown; one opened while shutting down is aborted at
+     * once.
+     */
+    protected openLease(lease: RunLease): void {
+        this.#leases.add(lease);
+        if (this.#stopping) {
+            lease.controller.abort(
+                new JobLeaseLostError('server shutting down'),
+            );
+        }
+    }
+
+    /** Stop tracking `lease`; its `close` calls this. */
+    protected closeLease(lease: RunLease): void {
+        this.#leases.delete(lease);
+    }
+
+    protected reportFailure(
+        name: string,
+        failures: number,
+        error: unknown,
+    ): void {
+        console.error(`[jobs] ${name} failed (${failures} in a row)`, error);
+        if (failures < JOB_FAILING_ALARM_AT) return;
+        this.clients.alarm.create(
+            `scheduled_job_failing:${this.namespace}:${name}`,
+            `System job ${name} keeps failing`,
+            {
+                namespace: this.namespace,
+                name,
+                failures,
+                error: asError(error),
+            },
+            'warning',
+            { dedup: true },
+        );
+    }
+}
+
+/**
+ * One hash slot per job. Neither part may contain `:` (`JOB_ID_PATTERN`), so no
+ * two (namespace, name) pairs share keys.
+ */
+const regionKeys = (namespace: string, name: string): RegionKeys => {
+    const tag = `{${namespace}:${name}}`;
+    return {
+        lock: `jobs:${tag}:lock`,
+        gate: `jobs:${tag}:next`,
+        run: `jobs:${tag}:run`,
+    };
+};

src/backend/services/jobs/jobRun.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+// What one job run needs, whichever scheduler holds the job. Also used by
+// extensions that replace `services.jobs`.
+
+import type { JobDefinition } from './JobScheduler.js';
+
+export const JOB_STATE_MAX_BYTES = 64 * 1024;
+export const JOB_ERROR_MAX_CHARS = 1024;
+
+/** The run no longer holds its job; another node may already be running it. */
+export class JobLeaseLostError extends Error {}
+
+export type RegisteredJob = Required<JobDefinition>;
+
+export type RunOutcome = 'success' | 'failure' | 'interrupted';
+
+/** What a run holds the job by. Its deadline is measured from before the claim. */
+export interface RunLease {
+    controller: AbortController;
+    /**
+     * Renews if little lease is left; false (and aborted) if the run can't
+     * start.
+     */
+    ready: () => Promise<boolean>;
+    close: () => void;
+}
+
+export const lostLease = (lease: RunLease, name: string): never => {
+    const lost = new JobLeaseLostError(`job ${name} lost its lease`);
+    lease.controller.abort(lost);
+    throw lost;
+};
+
+/** A run cut short by its signal is interrupted, whether or not it threw. */
+export const runOutcome = (signal: AbortSignal, error: unknown): RunOutcome => {
+    if (signal.aborted) return 'interrupted';
+    return error === null ? 'success' : 'failure';
+};
+
+export const parseState = (raw: string | null): unknown => {
+    if (raw === null) return null;
+    try {
+        return JSON.parse(raw);
+    } catch {
+        return null;
+    }
+};
+
+export const serializeState = (state: unknown): string => {
+    const json = JSON.stringify(state ?? null);
+    if (Buffer.byteLength(json) > JOB_STATE_MAX_BYTES) {
+        throw new Error(`job state is over ${JOB_STATE_MAX_BYTES} bytes`);
+    }
+    return json;
+};
+
+export const asError = (error: unknown): Error =>
+    error instanceof Error ? error : new Error(String(error));
+
+export const errorText = (error: unknown): string =>
+    (error instanceof Error ? error.message : String(error)).slice(
+        0,
+        JOB_ERROR_MAX_CHARS,
+    );

src/backend/services/jobs/jobsTestUtil.ts

@@ -17,24 +17,24 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-async function changeLanguage (lang) {
-    if ( !lang || window.locale === lang ) return;
-    // Shell strings are rendered once. Persist first, then reload.
-    try {
-        await puter.kv.set('user_preferences.language', lang);
-    } catch (err) {
-        console.warn('Could not save user_preferences.language:', err);
-        return;
-    }
-    window.locale = lang;
-    try {
-        window.mutate_user_preferences({
-            language: lang,
-        });
-    } catch (err) {
-        console.warn('Could not apply user_preferences.language:', err);
-    }
-    location.reload();
-}
+import type { JobRunContext } from './JobScheduler.js';
 
-export default changeLanguage;
+/**
+ * What a system job's `run` receives, for calling it directly: `saved` records
+ * each `saveState`, and `abort` aborts the signal.
+ */
+export const testJobContext = (
+    state: unknown = null,
+): JobRunContext & { saved: unknown[]; abort: () => void } => {
+    const controller = new AbortController();
+    const saved: unknown[] = [];
+    return {
+        signal: controller.signal,
+        state,
+        saveState: async (next) => {
+            saved.push(next);
+        },
+        saved,
+        abort: () => controller.abort(),
+    };
+};

src/backend/services/jobs/systemJobs.test.ts

@@ -0,0 +1,273 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The backend's own periodic jobs on the default scheduler: what registers,
+ * that each runs once across nodes sharing Redis, and that a single self-hosted
+ * node (sqlite, in-process Redis) runs every one of them.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import {
+    JobScheduler,
+    type JobDefinition,
+    type JobRunContext,
+} from './JobScheduler.js';
+
+type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
+
+const MINUTE = 60_000;
+const HOUR = 60 * MINUTE;
+
+const EXPECTED_JOBS = {
+    'auth.sessions.purge': {
+        every: 7 * 24 * HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 30 * MINUTE,
+    },
+    'kvShare.handles.purgeRevoked': {
+        every: 24 * HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'notifications.retention': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'metering.globalRateCheck': {
+        every: 25 * MINUTE,
+        scope: 'fleet',
+        nearPrimary: false,
+        leaseMs: 5 * MINUTE,
+    },
+    'events.subscriptions.expirySweep': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'events.userHandlers.gc': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'events.subscriptions.creditSweep': {
+        every: 15 * MINUTE,
+        scope: 'fleet',
+        nearPrimary: false,
+        leaseMs: 10 * MINUTE,
+    },
+};
+const JOB_NAMES = Object.keys(EXPECTED_JOBS);
+
+/** A timer armed while the server booted, and the frame that armed it. */
+interface ArmedTimer {
+    kind: 'interval' | 'timeout';
+    ms: number | undefined;
+    caller: string;
+}
+
+let server: PuterServer;
+const registered = new Map<string, JobDefinition>();
+const armed: ArmedTimer[] = [];
+
+/** The first stack frame outside this file and node_modules. */
+const callerOf = (stack: string | undefined): string =>
+    (stack ?? '')
+        .split('\n')
+        .slice(1)
+        .find(
+            (line) =>
+                !line.includes('systemJobs.test') &&
+                !line.includes('node_modules'),
+        ) ?? '';
+
+beforeAll(async () => {
+    const register = vi.spyOn(JobScheduler.prototype, 'register');
+    const realInterval = globalThis.setInterval;
+    const realTimeout = globalThis.setTimeout;
+    const intervals = vi.spyOn(globalThis, 'setInterval').mockImplementation(((
+        ...args: Parameters<typeof setInterval>
+    ) => {
+        armed.push({
+            kind: 'interval',
+            ms: args[1],
+            caller: callerOf(new Error().stack),
+        });
+        return realInterval(...args);
+    }) as never);
+    const timeouts = vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+        ...args: Parameters<typeof setTimeout>
+    ) => {
+        armed.push({
+            kind: 'timeout',
+            ms: args[1],
+            caller: callerOf(new Error().stack),
+        });
+        return realTimeout(...args);
+    }) as never);
+    try {
+        server = await setupTestServer({
+            events: { enabled: true },
+            notificationRetentionDays: 14,
+            maxGlobalUsagePerMinute: 1e15,
+            // A self-hosted node: nothing turns its jobs off.
+            jobs: { enabled: true },
+        } as IConfig);
+    } finally {
+        intervals.mockRestore();
+        timeouts.mockRestore();
+    }
+    for (const [name, definition] of register.mock.calls)
+        registered.set(name, definition);
+    register.mockRestore();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+const keysOf = (namespace: string, name: string) => ({
+    lock: `jobs:{${namespace}:${name}}:lock`,
+    gate: `jobs:{${namespace}:${name}}:next`,
+    run: `jobs:{${namespace}:${name}}:run`,
+});
+
+/** A node of its own in `region`, on the test server's Redis. */
+const node = (namespace: string, region: string): JobScheduler =>
+    new JobScheduler(
+        {
+            domain: 'puter.test',
+            serverId: `node-${region}`,
+            jobs: { namespace },
+        } as IConfig,
+        server.clients as SchedulerArgs[1],
+        server.stores as SchedulerArgs[2],
+        {
+            broadcast: { regionId: region },
+        } as unknown as SchedulerArgs[3],
+    );
+
+describe('system jobs', () => {
+    it('registers every moved job with its real cadence, scope and lease', () => {
+        expect(
+            Object.fromEntries(
+                [...registered].map(([name, job]) => [
+                    name,
+                    {
+                        every: job.every,
+                        scope: job.scope,
+                        nearPrimary: job.nearPrimary ?? false,
+                        leaseMs: job.leaseMs,
+                    },
+                ]),
+            ),
+        ).toEqual(EXPECTED_JOBS);
+    });
+
+    it('leaves no per-node timer behind for a moved job', () => {
+        const armedFrom = (file: string) =>
+            armed
+                .filter(({ caller }) => caller.includes(file))
+                .map(({ kind, ms }) => ({ kind, ms }));
+
+        expect(armedFrom('services/auth/AuthService')).toEqual([]);
+        expect(armedFrom('services/notification/NotificationService')).toEqual(
+            [],
+        );
+        // Node-local: the usage buffer flush.
+        expect(armedFrom('services/metering/MeteringService')).toEqual([
+            { kind: 'interval', ms: 10_000 },
+        ]);
+        // Node-local: the share-job runner (DB claims per job), and its kick.
+        expect(armedFrom('services/kvShare/KvShareService')).toEqual([
+            { kind: 'timeout', ms: expect.any(Number) },
+            { kind: 'interval', ms: MINUTE },
+        ]);
+        // Node-local: the region's pending-delivery sweep.
+        expect(armedFrom('services/events/EventsService')).toEqual([
+            { kind: 'interval', ms: 10_000 },
+        ]);
+    });
+
+    it.each(JOB_NAMES)(
+        'runs %s once across three nodes sharing Redis',
+        async (name) => {
+            const namespace = `ns-${randomUUID()}`;
+            const nodes = ['n1', 'n2', 'n3'].map((id) => node(namespace, id));
+            const job = registered.get(name)!;
+            let runs = 0;
+            for (const scheduler of nodes)
+                scheduler.register(name, {
+                    ...job,
+                    run: async (ctx: JobRunContext) => {
+                        runs += 1;
+                        await job.run(ctx);
+                    },
+                });
+
+            await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+            await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+
+            expect(runs).toBe(1);
+            const redis = server.clients.redis;
+            const keys = keysOf(namespace, name);
+            // A failure would have cut the gate to a 2 s backoff.
+            expect(await redis.pttl(keys.gate)).toBeGreaterThan(
+                job.every - MINUTE,
+            );
+            expect(await redis.hget(keys.run, 'failures')).toBeNull();
+        },
+    );
+
+    it('runs every job on a single self-hosted node, without the old per-job gates', async () => {
+        const jobs = server.services.jobs;
+        const redis = server.clients.redis;
+
+        await jobs.poll();
+
+        // The node's own poll timer may have started some of them first.
+        await vi.waitFor(
+            async () => {
+                for (const name of JOB_NAMES) {
+                    const keys = keysOf(jobs.namespace, name);
+                    expect(await redis.exists(keys.gate), name).toBe(1);
+                    // Released once the run is over.
+                    expect(await redis.exists(keys.lock), name).toBe(0);
+                    expect(await redis.hget(keys.run, 'failures'), name).toBe(
+                        null,
+                    );
+                }
+            },
+            { timeout: 10_000, interval: 100 },
+        );
+        await expect(
+            redis.exists('sessions:purge:gate', 'kvShare:purge:gate'),
+        ).resolves.toBe(0);
+    });
+});

src/backend/services/kvShare/KvShareService.ts

@@ -371,11 +371,9 @@ const SHARE_CHANGE_LOCK_RETRY_MS = 50;
 
 /** Retired handles are kept this long for their owner's listing, then deleted. */
 export const KV_SHARE_REVOKED_RETENTION_SECONDS = 30 * 24 * 60 * 60;
-// Every node checks hourly; the gate lets one node through per day.
-const PURGE_CHECK_MS = 60 * 60 * 1000;
-const PURGE_INITIAL_DELAY_MS = 5 * 60 * 1000;
-const PURGE_GATE_KEY = 'kvShare:purge:gate';
-const PURGE_GATE_SECONDS = 24 * 60 * 60;
+const PURGE_JOB = 'kvShare.handles.purgeRevoked';
+const PURGE_EVERY_MS = 24 * 60 * 60 * 1000;
+const PURGE_LEASE_MS = 15 * 60 * 1000;
 // Rows per delete, with a pause between, so replicas keep up.
 const PURGE_BATCH = 500;
 const PURGE_PAUSE_MS = 50;
@@ -536,7 +534,17 @@ export class KvShareService extends PuterService {
     #jobCursor: string | undefined;
 
     override onServerStart(): void {
-        this.#armPurge();
+        this.services.jobs.register(PURGE_JOB, {
+            every: PURGE_EVERY_MS,
+            scope: 'fleet',
+            nearPrimary: true,
+            leaseMs: PURGE_LEASE_MS,
+            run: async ({ signal }) => {
+                const purged = await this.purgeRevokedHandles(signal);
+                if (purged)
+                    console.log(`[kvShare] purged ${purged} retired handles`);
+            },
+        });
         this.#armJobs();
 
         // A check at one level is answered by a grant at any level above it.
@@ -607,51 +615,24 @@ export class KvShareService extends PuterService {
     // -- Retention purge ---------------------------------------------
 
     /**
-     * Delete handles retired longer ago than the retention, in batches. Returns
-     * the rows deleted, or null when another node holds today's gate. A missing
-     * row reads like a retired one everywhere a handle is resolved.
+     * Delete handles retired longer ago than the retention, in batches, until
+     * none are left or `signal` aborts. Returns the rows deleted. A missing row
+     * reads like a retired one everywhere a handle is resolved.
      */
-    async purgeRevokedHandles(): Promise<number | null> {
-        const claimed = await this.clients.redis.set(
-            PURGE_GATE_KEY,
-            String(Date.now()),
-            'EX',
-            PURGE_GATE_SECONDS,
-            'NX',
-        );
-        if (claimed !== 'OK') return null;
-
+    async purgeRevokedHandles(signal: AbortSignal): Promise<number> {
         const cutoff =
             Math.floor(Date.now() / 1000) - KV_SHARE_REVOKED_RETENTION_SECONDS;
         let purged = 0;
-        let finished = false;
-        try {
-            for (let i = 0; i < PURGE_MAX_BATCHES; i++) {
-                if (this.#stopping) return purged;
-                const batch =
-                    await this.stores.kvShareHandle.deleteRevokedBefore(
-                        cutoff,
-                        PURGE_BATCH,
-                    );
-                purged += batch;
-                if (batch < PURGE_BATCH) break;
-                await new Promise((resolve) =>
-                    setTimeout(resolve, PURGE_PAUSE_MS),
-                );
-            }
-            finished = true;
-            return purged;
-        } finally {
-            // A run cut short frees the gate, so another node picks it up on
-            // its next check instead of tomorrow.
-            if (!finished) {
-                try {
-                    await this.clients.redis.del(PURGE_GATE_KEY);
-                } catch {
-                    // The gate expires on its own.
-                }
-            }
+        for (let i = 0; i < PURGE_MAX_BATCHES && !signal.aborted; i++) {
+            const batch = await this.stores.kvShareHandle.deleteRevokedBefore(
+                cutoff,
+                PURGE_BATCH,
+            );
+            purged += batch;
+            if (batch < PURGE_BATCH) break;
+            await new Promise((resolve) => setTimeout(resolve, PURGE_PAUSE_MS));
         }
+        return purged;
     }
 
     /**
@@ -1047,27 +1028,6 @@ export class KvShareService extends PuterService {
         this.#timers.push(kick, timer);
     }
 
-    #armPurge(): void {
-        const run = async () => {
-            try {
-                const purged = await this.purgeRevokedHandles();
-                if (purged)
-                    console.log(`[kvShare] purged ${purged} retired handles`);
-            } catch (err) {
-                console.warn('[kvShare] retired-handle purge failed', err);
-            }
-        };
-        // Jittered so a deploy does not have every node check at once.
-        const kick = setTimeout(
-            () => void run(),
-            PURGE_INITIAL_DELAY_MS * (0.5 + Math.random()),
-        );
-        kick.unref?.();
-        const timer = setInterval(() => void run(), PURGE_CHECK_MS);
-        timer.unref?.();
-        this.#timers.push(kick, timer);
-    }
-
     /** Whether the level a mint asks for is available on this server. */
     #assertAccessEnabled(access: KvShareAccess): void {
         if (access === 'watch') {
@@ -1745,11 +1705,14 @@ export class KvShareService extends PuterService {
      * The other party of a share, for a notification one party sends through
      * its handle from the app whose data it is. Every check runs before the
      * pair budget is spent. A handle that is unknown, retired, not the
-     * caller's, or seen from another app reads as absent.
+     * caller's, or seen from another app reads as absent. `spendBudget: false`
+     * checks without spending, for a send that is only being scheduled. Called
+     * by extensions.
      */
     async resolveNotifyRecipient(
         actor: Actor,
         handle: unknown,
+        { spendBudget = true }: { spendBudget?: boolean } = {},
     ): Promise<KvShareNotifyTarget> {
         const named = typeof handle === 'string' ? handle : '';
         const senderId = actor.user?.id;
@@ -1782,7 +1745,8 @@ export class KvShareService extends PuterService {
         if (!checked?.holds) throw unknownKvShareHandle(named);
         if (checked.refuses) throw recipientNotAcceptingShares();
 
-        await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
+        if (spendBudget)
+            await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
         return {
             recipientUserId: recipientId,
             appUid: share.appUid,

src/backend/services/metering/MeteringService.ts

@@ -111,6 +111,11 @@ type UsageWriteResult = {
     detail?: FlatUsageDetail;
 };
 
+/** Compares fleet-wide spend against `maxGlobalUsagePerMinute`. */
+const GLOBAL_RATE_CHECK_JOB = 'metering.globalRateCheck';
+const GLOBAL_RATE_CHECK_EVERY_MS = 25 * 60 * 1000;
+const GLOBAL_RATE_CHECK_LEASE_MS = 5 * 60 * 1000;
+
 // -- Helpers ----------------------------------------------------------
 
 /**
@@ -284,7 +289,6 @@ export class MeteringService extends PuterService {
      */
     static APP_TOTALS_LIST_LIMIT = 4_000;
 
-    private rateCheckTimer: ReturnType<typeof setInterval> | null = null;
     private usageBufferTimer: ReturnType<typeof setInterval> | null = null;
     private extraPolicies: SubscriptionPolicy[] = [];
     private subscriptionResolvers: SubscriptionResolver[] = [];
@@ -415,15 +419,15 @@ export class MeteringService extends PuterService {
             },
         );
 
-        this.rateCheckTimer = setInterval(
-            () => {
-                this.checkRateOfChange().catch((e) => {
-                    console.error('[metering] rate-of-change check failed', e);
-                });
-            },
-            1000 * 60 * 25,
-        );
-        this.rateCheckTimer.unref?.();
+        const maxPerMinute = this.config.maxGlobalUsagePerMinute;
+        if (maxPerMinute && maxPerMinute > 0) {
+            this.services.jobs.register(GLOBAL_RATE_CHECK_JOB, {
+                every: GLOBAL_RATE_CHECK_EVERY_MS,
+                scope: 'fleet',
+                leaseMs: GLOBAL_RATE_CHECK_LEASE_MS,
+                run: () => this.checkRateOfChange(),
+            });
+        }
 
         const flushInterval =
             this.config.meteringUsageBufferFlushMs &&
@@ -454,10 +458,6 @@ export class MeteringService extends PuterService {
     }
 
     override async onServerShutdown(): Promise<void> {
-        if (this.rateCheckTimer) {
-            clearInterval(this.rateCheckTimer);
-            this.rateCheckTimer = null;
-        }
         if (this.usageBufferTimer) {
             clearInterval(this.usageBufferTimer);
             this.usageBufferTimer = null;
@@ -2948,6 +2948,8 @@ export class MeteringService extends PuterService {
             timestamp: number;
         } | null;
 
+        // Several schedulers can share the baseline (one per region without a
+        // fleet scheduler, or old nodes mid-rollout); keep windows wide.
         if (lastChange && now - lastChange.timestamp <= 14 * 60 * 1000) return;
 
         const globalUsage = await this.getGlobalUsage();

src/backend/services/notification/NotificationService.test.ts

@@ -395,6 +395,36 @@ describe('NotificationService.notifyUpdate', () => {
         written.stop();
     });
 
+    it('sends the desktop rewrite before reading the row back, so a failed read still updates it', async () => {
+        const user = await makeUser();
+        const uid = await notifications.notify(
+            [user.id],
+            { title: 'one share' },
+            { type: 'share.received' },
+        );
+        const pushed = collect('outer.gui.notif.message');
+        const read = vi
+            .spyOn(server.stores.notification, 'getByUid')
+            .mockRejectedValueOnce(new Error('replica down'));
+        try {
+            await expect(
+                notifications.notifyUpdate(
+                    uid,
+                    user.id,
+                    { title: 'two shares' },
+                    { type: 'share.received' },
+                ),
+            ).rejects.toThrow('replica down');
+            expect(pushed.seen).toHaveLength(1);
+            expect((pushed.seen[0] as NotifPush).response.notification).toEqual(
+                { title: 'two shares', type: 'share.received' },
+            );
+        } finally {
+            read.mockRestore();
+            pushed.stop();
+        }
+    });
+
     it('refuses to fold a type that does not group', async () => {
         const user = await makeUser();
         await expect(
@@ -882,17 +912,42 @@ describe('NotificationService.sweepExpired', () => {
     });
 
     it('stops at the pass cap and leaves the rest for the next sweep', async () => {
+        // A backlog that always fills the batch: the cap, not a short batch,
+        // is what ends the call. Seeding 500k rows would take minutes.
+        const deletes = vi
+            .spyOn(server.stores.notification, 'deleteCreatedBefore')
+            .mockResolvedValue(500);
+        try {
+            expect(await notifications.sweepExpired()).toBe(500_000);
+            expect(deletes).toHaveBeenCalledTimes(1000);
+        } finally {
+            deletes.mockRestore();
+        }
+    });
+
+    it('stops between batches once its signal aborts', async () => {
         await notifications.sweepExpired();
         const user = await makeUser();
-        // 50 passes * 500/batch = 25,000 — a backlog past that so the cap,
-        // not a short batch, is what ends the first call.
-        await seedExpired(user.id, 25_050);
-
-        expect(await notifications.sweepExpired()).toBe(25_000);
-        expect(
-            await server.stores.notification.listByUserId(user.id),
-        ).toHaveLength(50);
-        expect(await notifications.sweepExpired()).toBe(50);
+        await seedExpired(user.id, 1200);
+        const controller = new AbortController();
+        const deleteBatch = server.stores.notification.deleteCreatedBefore.bind(
+            server.stores.notification,
+        );
+        const spy = vi
+            .spyOn(server.stores.notification, 'deleteCreatedBefore')
+            .mockImplementation(async (days: number, limit: number) => {
+                const removed = await deleteBatch(days, limit);
+                controller.abort();
+                return removed;
+            });
+        try {
+            expect(await notifications.sweepExpired(controller.signal)).toBe(
+                500,
+            );
+        } finally {
+            spy.mockRestore();
+        }
+        expect(await notifications.sweepExpired()).toBe(700);
     });
 
     it('sweeps nothing when no retention is configured', async () => {

src/backend/services/notification/NotificationService.ts

@@ -48,12 +48,16 @@ export type {
 } from './notificationTypes.js';
 export { canViewNotification } from './notificationAudience.js';
 
-/** How often the retention sweep runs. */
-const RETENTION_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
+const RETENTION_SWEEP_JOB = 'notifications.retention';
+const RETENTION_SWEEP_EVERY_MS = 60 * 60 * 1000;
+const RETENTION_SWEEP_LEASE_MS = 15 * 60 * 1000;
 /** Rows one delete takes. Small enough not to hold a lock anyone waits on. */
 const RETENTION_BATCH_SIZE = 500;
-/** Batches one sweep takes, so a large backlog drains over several passes. */
-const RETENTION_MAX_BATCHES = 50;
+/**
+ * Batches one sweep takes at most. It runs once for the fleet, so this is the
+ * fleet's hourly drain; the job's signal stops it sooner.
+ */
+const RETENTION_MAX_BATCHES = 1000;
 /** Unread notifications one reconnect replays. */
 const UNREAD_REPLAY_LIMIT = 200;
 
@@ -69,11 +73,20 @@ const UNREAD_REPLAY_LIMIT = 200;
 export class NotificationService extends PuterService {
     /** User.id → debounce timeout */
     #connectTimeouts = new Map<number, ReturnType<typeof setTimeout>>();
-    #retentionSweep: ReturnType<typeof setInterval> | null = null;
     #adapter: NotificationSocketAdapter | null = null;
 
     override onServerStart(): void {
-        this.#armRetentionSweep();
+        if (this.#retentionDays() > 0) {
+            this.services.jobs.register(RETENTION_SWEEP_JOB, {
+                every: RETENTION_SWEEP_EVERY_MS,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: RETENTION_SWEEP_LEASE_MS,
+                run: async ({ signal }) => {
+                    await this.sweepExpired(signal);
+                },
+            });
+        }
         this.#socket().attach();
 
         // When a user opens the GUI, send their pending unreads.
@@ -103,11 +116,6 @@ export class NotificationService extends PuterService {
         );
     }
 
-    override onServerPrepareShutdown(): void {
-        if (this.#retentionSweep) clearInterval(this.#retentionSweep);
-        this.#retentionSweep = null;
-    }
-
     // -- Public API --------------------------------------------------
 
     /**
@@ -176,25 +184,17 @@ export class NotificationService extends PuterService {
                 }
 
                 if (!opts.silent) {
-                    // Listeners get their own copy; the socket sends `payload`.
-                    this.clients.event.emit(
-                        'notif.written',
+                    await this.emitWritten(
                         {
                             userId,
                             uid,
                             type: registered.type,
                             audience: registered.audience,
                             appUid,
-                            value: structuredClone(payload),
-                            sendId: uuidv4(),
+                            value: payload,
                         },
-                        {},
+                        { sendId: uuidv4() },
                     );
-                    await this.#push(userId, uid, payload, {
-                        type: registered.type,
-                        audience: registered.audience,
-                        appUid,
-                    });
                 }
                 return uid;
             }),
@@ -241,50 +241,107 @@ export class NotificationService extends PuterService {
         if (!updated) return false;
         if (opts.silent) return true;
 
+        // Before the read, so the desktop copy updates even if it fails.
         const desktop = inAccountMailbox(registered.type);
-        if (desktop) {
-            // Same uid as the original: the client replaces what it is already
-            // showing rather than stacking another copy. It goes straight to
-            // the wire rather than through dispatch — the row was already
-            // published, and republishing it under its own id is a duplicate
-            // to anything deduplicating on `event.id`.
-            this.#socket().message(userId, uid, payload);
-            this.#markDelivered(uid, userId);
-        }
+        if (desktop) this.#sendRewrite(userId, uid, payload);
         const row = await this.stores.notification.getByUid(uid, { userId });
         const app = notification.app as { uid?: unknown } | undefined;
         const appUid =
             row?.app_uid ?? (typeof app?.uid === 'string' ? app.uid : null);
-        this.clients.event.emit(
-            'notif.written',
+        await this.emitWritten(
             {
                 userId,
                 uid,
                 type: registered.type,
                 audience: registered.audience,
                 appUid,
-                value: structuredClone(payload),
+                value: payload,
                 updated: true,
-                sendId: uuidv4(),
+                replacedAt,
+            },
+            { sendId: uuidv4(), rewriteSent: desktop },
+        );
+        return true;
+    }
+
+    /**
+     * Deliver a notification already written: `notif.written` for listeners,
+     * then the live copy. `sendId` names this one send; a delayed send passes
+     * the same one each time it delivers, so a listener can tell a repeat. An
+     * `updated` row is an earlier notification rewritten in place, and
+     * `replacedAt` marks it a replacement. `handlerDepth` is the depth the
+     * write was made at, when no actor in context carries it. `rewriteSent`
+     * says the caller already sent a desktop rewrite. `replayable` marks a send
+     * that may be emitted again under the same `sendId`. Called by extensions.
+     */
+    async emitWritten(
+        written: {
+            userId: number;
+            uid: string;
+            type: string;
+            audience: string;
+            appUid: string | null;
+            value: Record<string, unknown>;
+            updated?: boolean;
+            replacedAt?: number;
+        },
+        opts: {
+            sendId: string;
+            handlerDepth?: number;
+            rewriteSent?: boolean;
+            replayable?: boolean;
+        },
+    ): Promise<void> {
+        const { userId, uid, value: payload, updated } = written;
+        const scope = {
+            type: written.type,
+            audience: written.audience,
+            appUid: written.appUid,
+        };
+        const desktop = inAccountMailbox(scope.type);
+        if (updated && desktop && !opts.rewriteSent)
+            this.#sendRewrite(userId, uid, payload);
+        // Listeners get their own copy; the socket sends `payload`.
+        this.clients.event.emit(
+            'notif.written',
+            {
+                userId,
+                uid,
+                ...scope,
+                value: structuredClone(payload),
+                ...(updated ? { updated: true as const } : {}),
+                sendId: opts.sendId,
+                ...(opts.replayable ? { replayable: true as const } : {}),
             },
             {},
         );
-        // A type kept off the desktop has dispatch as its only live path, so
-        // the rewrite goes out there again, under the same uid.
-        if (!desktop) {
+        // A rewrite of a type kept off the desktop has dispatch as its only
+        // live path, so it goes out there again, under the same uid.
+        if (!updated || !desktop)
             await this.#push(
                 userId,
                 uid,
                 payload,
-                {
-                    type: registered.type,
-                    audience: registered.audience,
-                    appUid,
-                },
-                replacedAt,
+                scope,
+                written.replacedAt,
+                opts.handlerDepth,
             );
-        }
-        return true;
+    }
+
+    /**
+     * Same uid as the original: the client replaces what it is already showing
+     * rather than stacking another copy. It goes straight to the wire rather
+     * than through dispatch — the row was already published, and republishing
+     * it under its own id is a duplicate to anything deduplicating on
+     * `event.id`.
+     */
+    #sendRewrite(
+        userId: number,
+        uid: string,
+        payload: Record<string, unknown>,
+    ): void {
+        this.#socket().message(userId, uid, payload);
+        this.#markDelivered(uid, userId);
     }
 
     /**
@@ -348,17 +405,15 @@ export class NotificationService extends PuterService {
      * Drop notifications past the retention window, in batches, and report how
      * many went. Deleting is all there is to it: nothing is pushed, because a
      * two-week-old row is not news, and a client listing again simply stops
-     * seeing it.
-     *
-     * Every node sweeps. Batches are small and the delete is idempotent, so two
-     * nodes overlapping costs a few empty batches, not correctness.
+     * seeing it. Stops between batches once `signal` aborts.
      */
-    async sweepExpired(): Promise<number> {
+    async sweepExpired(signal?: AbortSignal): Promise<number> {
         const days = this.#retentionDays();
         if (days <= 0) return 0;
 
         let removed = 0;
         for (let pass = 0; pass < RETENTION_MAX_BATCHES; pass++) {
+            if (signal?.aborted) break;
             const batch = await this.stores.notification.deleteCreatedBefore(
                 days,
                 RETENTION_BATCH_SIZE,
@@ -411,6 +466,7 @@ export class NotificationService extends PuterService {
         payload: Record<string, unknown>,
         scope: { type: string; audience: string; appUid: string | null },
         replacedAt?: number,
+        depth?: number,
     ): Promise<void> {
         const desktop = inAccountMailbox(scope.type);
         if (!notificationsFoldInEnabled(this.config)) {
@@ -430,9 +486,8 @@ export class NotificationService extends PuterService {
             return;
         }
 
-        const handlerDepth = handlerDepthOf(
-            Context.get('actor') as Actor | undefined,
-        );
+        const handlerDepth =
+            depth ?? handlerDepthOf(Context.get('actor') as Actor | undefined);
         this.clients.event.emit(
             'notif.created',
             {
@@ -470,17 +525,6 @@ export class NotificationService extends PuterService {
         return Number.isFinite(configured) && configured > 0 ? configured : 0;
     }
 
-    #armRetentionSweep(): void {
-        if (this.#retentionDays() <= 0) return;
-        const sweep = setInterval(() => {
-            void this.sweepExpired().catch((err) => {
-                console.warn('[notification] retention sweep failed', err);
-            });
-        }, RETENTION_SWEEP_INTERVAL_MS);
-        sweep.unref?.();
-        this.#retentionSweep = sweep;
-    }
-
     /**
      * What a reconnecting client missed. One statement marks the batch shown —
      * a per-row update is a round trip per notification on a path that runs

src/backend/services/notification/appMessage.test.ts

@@ -243,4 +243,61 @@ describe('appMessageSender', () => {
             'a'.repeat(120),
         );
     });
+
+    describe('an app made for an origin', () => {
+        const uid = 'app-65d98228-cfa5-5d18-a442-f6eeaf3dcfc7';
+        const originApp = (fields: Record<string, unknown> = {}) => ({
+            uid,
+            name: uid,
+            title: uid,
+            description: 'App created from origin https://chat.example.com',
+            index_url: 'https://chat.example.com',
+            ...fields,
+        });
+
+        it('is named by the hostname of its index_url, without a port', () => {
+            expect(appMessageSender(uid, originApp()).name).toBe(
+                'chat.example.com',
+            );
+            expect(
+                appMessageSender(
+                    uid,
+                    originApp({
+                        index_url: 'https://chat.example.com:8443/app/',
+                    }),
+                ).name,
+            ).toBe('chat.example.com');
+        });
+
+        it('keeps its stored name when its index_url does not parse', () => {
+            expect(
+                appMessageSender(uid, originApp({ index_url: 'not a url' }))
+                    .name,
+            ).toBe(uid);
+            expect(
+                appMessageSender(uid, originApp({ index_url: undefined })).name,
+            ).toBe(uid);
+        });
+
+        it('keeps a title its developer set', () => {
+            expect(
+                appMessageSender(uid, originApp({ title: 'Chat' })).name,
+            ).toBe('Chat');
+        });
+
+        it('leaves apps that were not made for an origin alone', () => {
+            expect(
+                appMessageSender(uid, originApp({ description: 'My chat app' }))
+                    .name,
+            ).toBe(uid);
+            expect(
+                appMessageSender('app-1', {
+                    uid: 'app-1',
+                    name: 'chat',
+                    title: 'Chat',
+                    index_url: 'https://chat.example.com',
+                }).name,
+            ).toBe('Chat');
+        });
+    });
 });

src/backend/services/notification/appMessage.ts

@@ -18,6 +18,7 @@
  */
 
 import { HttpError } from '../../core/http/HttpError.js';
+import { isOriginBootstrapApp } from '../../util/appView.js';
 
 /** The one notification type an app may create. */
 export const APP_MESSAGE_TYPE = 'app.message';
@@ -60,14 +61,35 @@ export type AppMessage = {
 
 export type AppMessageSender = { uid: string; name?: string; icon?: string };
 
+/** The hostname an app made for an origin is named by. */
+const originAppHost = (row: unknown): string | undefined => {
+    if (!isOriginBootstrapApp(row) || typeof row.index_url !== 'string') {
+        return undefined;
+    }
+    try {
+        return new URL(row.index_url).hostname || undefined;
+    } catch {
+        return undefined;
+    }
+};
+
 /** What a message says about its app, read off the app's row. */
 export const appMessageSender = (
     uid: string,
-    row: { name?: unknown; title?: unknown; icon?: unknown } | null | undefined,
+    row:
+        | {
+              name?: unknown;
+              title?: unknown;
+              icon?: unknown;
+              description?: unknown;
+              index_url?: unknown;
+          }
+        | null
+        | undefined,
 ): AppMessageSender => {
     const sender: AppMessageSender = { uid };
     // Cleaned before the empty check, so a title of only invisible characters still leaves a visible sender.
-    const name = [row?.title, row?.name]
+    const name = [originAppHost(row), row?.title, row?.name]
         .map((v) =>
             typeof v === 'string'
                 ? [...cleanText(v)]

src/backend/services/notification/notificationTypes.test.ts

@@ -3,18 +3,19 @@
  *
  * This file is part of Puter.
  *
- * Puter is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published
- * by the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
+ * Puter is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
  *
  * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ * along with this program. If not, see
+ * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/).
  */
 
 import { describe, expect, it } from 'vitest';
@@ -53,6 +54,9 @@ describe('NOTIFICATION_TYPES', () => {
             audience: 'developer',
             appScoped: true,
         });
+        expect(
+            findNotificationType('app.events.handler_suspended'),
+        ).toMatchObject({ audience: 'app-user', appScoped: true });
         expect(findNotificationType('app.events.ended')).toMatchObject({
             audience: 'app-user',
             appScoped: false,
@@ -61,9 +65,9 @@ describe('NOTIFICATION_TYPES', () => {
 
     it('projects a subject naming the app, or the recipient when there is none', () => {
         const account = findNotificationType('share.received')!;
-        expect(
-            account.subject({ userUuid: 'user-uuid', appUid: null }),
-        ).toBe('notif:user-uuid:account');
+        expect(account.subject({ userUuid: 'user-uuid', appUid: null })).toBe(
+            'notif:user-uuid:account',
+        );
 
         const developer = findNotificationType('app.events.suspended')!;
         expect(
@@ -130,18 +134,20 @@ describe('resolveNotificationWrite', () => {
     });
 
     it('requires an app uid for an app message', () => {
-        expect(resolveNotificationWrite('app.message', 'app-1')).toMatchObject(
-            { audience: 'app-user', appScoped: true, groupable: true },
-        );
+        expect(resolveNotificationWrite('app.message', 'app-1')).toMatchObject({
+            audience: 'app-user',
+            appScoped: true,
+            groupable: true,
+        });
         expect(() => resolveNotificationWrite('app.message', null)).toThrow(
             'requires an app uid',
         );
     });
 
     it('lets a subscription an account made end without naming an app', () => {
-        expect(resolveNotificationWrite('app.events.ended', null).audience).toBe(
-            'app-user',
-        );
+        expect(
+            resolveNotificationWrite('app.events.ended', null).audience,
+        ).toBe('app-user');
     });
 });
 

src/backend/services/notification/notificationTypes.ts

@@ -115,6 +115,15 @@ export const NOTIFICATION_TYPES = [
         groupable: false,
         subject: subjectFor('developer'),
     },
+    // A handler the user published into their own scope stopped taking
+    // deliveries: the code is theirs, so the notice is too.
+    {
+        type: 'app.events.handler_suspended',
+        audience: 'app-user',
+        appScoped: true,
+        groupable: false,
+        subject: subjectFor('app-user'),
+    },
     // The only type apps create; its kind goes in the value's `channel`. A
     // newer one replaces the newest with the same `group` unless that one was
     // shown or dismissed.

src/backend/services/types.ts

@@ -17,6 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+import type { ScheduleServiceContract } from '../core/schedules.js';
 import type { TeamServiceContract } from '../core/teams.js';
 
 import type { puterClients } from '../clients';
@@ -38,6 +39,8 @@ export interface IPuterServiceInstances {}
 export interface IExtensionServiceInstances {
     /** Teams ships separately; boot guarantees it when `teams_enabled` is on. */
     team?: TeamServiceContract;
+    /** Schedules ship separately; without them their routes answer 503. */
+    schedules?: ScheduleServiceContract;
     [key: string]: unknown;
 }
 

src/backend/services/user/SignupService.test.ts

@@ -381,3 +381,29 @@ describe('SignupService.confirmEmail', () => {
         expect(after!.email).toBe(other.email);
     });
 });
+
+describe('SignupService.validateEmail', () => {
+    it('refuses Puter mailbox addresses before any listener runs', async () => {
+        const listener = vi.fn();
+        server.clients.event.on('email.validate', listener);
+        try {
+            for (const email of [
+                '[email protected]',
+                '[email protected]',
+                '[email protected]',
+            ]) {
+                await expect(
+                    server.services.signup.validateEmail(email),
+                ).rejects.toMatchObject({ statusCode: 400 });
+            }
+            expect(listener).not.toHaveBeenCalled();
+            await expect(
+                server.services.signup.validateEmail(
+                    `ok-${slug()}@example.com`,
+                ),
+            ).resolves.toBeUndefined();
+        } finally {
+            server.clients.event.off('email.validate', listener);
+        }
+    });
+});

src/backend/services/user/SignupService.ts

@@ -26,6 +26,7 @@ import { HttpError } from '../../core/http/HttpError.js';
 import type { UserRow } from '../../stores/user/UserStore.js';
 import { isOwnedEmailConflict } from '../../stores/user/UserStore.js';
 import { cleanEmail, isBlockedEmail } from '../../util/email.js';
+import { isPuterEmailAddress } from '../email/mailbox.js';
 import {
     bonusCodeInvalidError,
     checkSignupBonus,
@@ -95,10 +96,20 @@ export const signupClientIp = (req: Request): string | null =>
  */
 export class SignupService extends PuterService {
     /**
-     * Config blocklist (suffix match on the cleaned address), then the
-     * `email.validate` hook. Throws 400 on rejection.
+     * Puter mailbox addresses, the config blocklist (suffix match on the
+     * cleaned address), then the `email.validate` hook. Throws 400 on
+     * rejection.
      */
     async validateEmail(email: string): Promise<void> {
+        // A Puter mailbox can't be an account's own email: it is reached
+        // through an account in the first place.
+        if (isPuterEmailAddress(email.trim())) {
+            throw new HttpError(
+                400,
+                'A Puter email address cannot be used as your account email. Please use a different email address.',
+                { legacyCode: 'email_not_allowed' as never },
+            );
+        }
         if (isBlockedEmail(email, this.config.blockedEmailDomains)) {
             throw new HttpError(400, 'This email is not allowed.', {
                 legacyCode: 'email_not_allowed' as never,

src/backend/stores/app/AppStore.js

@@ -262,6 +262,19 @@ export class AppStore extends PuterStore {
         return result;
     }
 
+    /**
+     * Cache-free existence check by uid against the primary. Called by
+     * extensions.
+     */
+    async uidExistsOnPrimary(uid) {
+        if (!uid) return false;
+        const rows = await this.clients.db.pread(
+            'SELECT `id` FROM `apps` WHERE `uid` = ? LIMIT 1',
+            [uid],
+        );
+        return rows.length > 0;
+    }
+
     /** Cache-free existence check against the primary. */
     async existsOnPrimary(appId) {
         if (appId === undefined || appId === null) return false;
@@ -788,7 +801,7 @@ export class AppStore extends PuterStore {
      * the row stays unclaimed until a developer registers an app for the same
      * origin and AppDriver's merge path takes ownership. Marker shape (`name
      * === uid && title === uid` + description prefix) is what
-     * `#isOriginBootstrapApp` checks.
+     * `isOriginBootstrapApp` (util/appView) checks.
      */
     async createFromOrigin(
         uid,

src/backend/stores/events/DurableSubscriptionStore.integration.test.ts

@@ -43,6 +43,7 @@ import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENT_SUBSCRIPTION_WIDTHS } from './columnWidths.js';
 import type { DurableSubscriptionInput } from './DurableSubscriptionStore.js';
+import { scheduleToken } from './types.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 
@@ -201,9 +202,9 @@ describe('creating a subscription', () => {
         const { row } = await durable().create(input());
 
         await expect(cache().userHasAny(userId)).resolves.toBe(true);
-        await expect(
-            cache().watchedTokens(userId, [token()]),
-        ).resolves.toEqual([token()]);
+        await expect(cache().watchedTokens(userId, [token()])).resolves.toEqual(
+            [token()],
+        );
         await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
             row,
         ]);
@@ -264,6 +265,114 @@ describe('creating a subscription', () => {
     });
 });
 
+describe('handler scope', () => {
+    const appUid = 'app-scoped';
+
+    it('round-trips through create and every select path', async () => {
+        const { row: user } = await durable().create(
+            input({ appUid, handlerName: 'h', handlerScope: 'user' }),
+        );
+        const { row: app } = await durable().create(
+            input({ appUid, handlerName: 'h' }),
+        );
+        expect(user.handlerScope).toBe('user');
+        expect(app).not.toHaveProperty('handlerScope');
+
+        await expect(durable().getBySubId(user.subId)).resolves.toMatchObject({
+            handlerScope: 'user',
+        });
+        await expect(
+            durable().getBySubId(app.subId),
+        ).resolves.not.toHaveProperty('handlerScope');
+        const listed = await durable().listForHolder(userId, { appUid });
+        expect(
+            new Map(listed.items.map((r) => [r.subId, r.handlerScope])),
+        ).toEqual(
+            new Map<string, string | undefined>([
+                [user.subId, 'user'],
+                [app.subId, undefined],
+            ]),
+        );
+        const stored = await env.server.clients.db.read(
+            'SELECT `sub_id`, `handler_scope` FROM `event_subscriptions` WHERE `app_uid` = ?',
+            [appUid],
+        );
+        expect(
+            Object.fromEntries(stored.map((r) => [r.sub_id, r.handler_scope])),
+        ).toEqual({ [user.subId]: 1, [app.subId]: 0 });
+    });
+
+    it('lists by scope and holder, never across scopes', async () => {
+        const { row: appRow } = await durable().create(
+            input({ appUid, handlerName: 'h' }),
+        );
+        const { row: mine } = await durable().create(
+            input({ appUid, handlerName: 'h', handlerScope: 'user' }),
+        );
+        const { row: theirs } = await durable().create(
+            input({
+                appUid,
+                handlerName: 'h',
+                handlerScope: 'user',
+                holderUserId: otherUserId,
+                ownerUserId: otherUserId,
+                token: `f#other-${uuidv4()}`,
+            }),
+        );
+        const ids = async (scope: 'app' | 'user', holder: number, name = 'h') =>
+            (await durable().listByHandler(appUid, scope, holder, name))
+                .map((r) => r.subId)
+                .sort();
+
+        expect(await ids('app', 0)).toEqual([appRow.subId]);
+        // Holder is ignored for app scope.
+        expect(await ids('app', userId)).toEqual([appRow.subId]);
+        expect(await ids('user', userId)).toEqual([mine.subId]);
+        expect(await ids('user', otherUserId)).toEqual([theirs.subId]);
+        expect(await ids('user', userId, 'missing')).toEqual([]);
+    });
+
+    it('lists the suspended half of a user scope', async () => {
+        const { row } = await durable().create(
+            input({ appUid, handlerName: 'h', handlerScope: 'user' }),
+        );
+        await env.server.clients.db.write(
+            'UPDATE `event_subscriptions` SET `suspended_at` = 1, `suspended_reason` = ? WHERE `sub_id` = ?',
+            ['handler_not_found', row.subId],
+        );
+        await expect(
+            durable().listByHandler(appUid, 'user', userId, 'h'),
+        ).resolves.toEqual([]);
+        await expect(
+            durable().listByHandler(appUid, 'user', userId, 'h', {
+                suspendedReason: 'handler_not_found',
+            }),
+        ).resolves.toEqual([expect.objectContaining({ subId: row.subId })]);
+    });
+
+    it('reads a user scope through the holder index', async () => {
+        const pread = vi.spyOn(env.server.clients.db, 'pread');
+        try {
+            await durable().listByHandler(appUid, 'user', userId, 'h');
+            const [statement, params] = pread.mock.calls[0]!;
+            const plan = await env.server.clients.db.read(
+                `EXPLAIN QUERY PLAN ${statement}`,
+                params,
+            );
+            const detail = plan.map((r) => String(r.detail)).join('\n');
+            expect(detail).toContain('idx_event_subscriptions_holder');
+            expect(detail).not.toContain('idx_event_subscriptions_app_handler');
+        } finally {
+            pread.mockRestore();
+        }
+    });
+
+    it('carries the scope through the schedule join', async () => {
+        const join = durable().scheduleSubscriptionJoin('s', 'x.`sub_id`');
+        expect(join.columns).toContain('`sub_handler_scope`');
+    });
+});
+
 describe('validation at the row write', () => {
     it('refuses a context past the hard cap', async () => {
         await expect(
@@ -468,7 +577,9 @@ describe('the per-account cap', () => {
             input({ expiresAt: Math.floor(Date.now() / 1000) - 60 }),
         );
         await durable().create(input());
-        await expect(durable().countForHolder(userId)).resolves.toMatchObject({ total: 1 });
+        await expect(durable().countForHolder(userId)).resolves.toMatchObject({
+            total: 1,
+        });
     });
 });
 
@@ -547,9 +658,13 @@ describe('rows suspended for good', () => {
         await durable().suspend([failing], 'failures');
         await durable().suspend([revoked], 'permission_revoked');
 
-        const held = await durable().listActiveForHolder(userId, null, {
-            includeSuspended: true,
-        });
+        const { rows: held } = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            0,
+            undefined,
+            { includeSuspended: true },
+        );
         expect(held.map((row) => row.subId).sort()).toEqual(
             [live.subId, failing.subId].sort(),
         );
@@ -565,7 +680,9 @@ describe('rows suspended for good', () => {
             },
         );
 
-        await durable().create(input());
+        const created = await durable().create(input());
+        // The trim's bump, for the caller to publish to the other regions.
+        expect(created.reaped.map((bump) => bump.userId)).toEqual([userId]);
 
         await expect(durable().countForHolder(userId)).resolves.toEqual({
             total: 1,
@@ -600,9 +717,9 @@ describe('removal', () => {
 
         await durable().remove(row);
 
-        await expect(
-            cache().watchedTokens(userId, [token()]),
-        ).resolves.toEqual([token()]);
+        await expect(cache().watchedTokens(userId, [token()])).resolves.toEqual(
+            [token()],
+        );
         await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
             survivor.row,
         ]);
@@ -744,9 +861,11 @@ describe('the expiry sweep', () => {
         // one back to prove the sweep is what removes it.
         await cache().cacheDurable([expired.row]);
 
-        await expect(durable().sweepExpired(500)).resolves.toBe(1);
+        expect((await durable().sweepExpired(500)).rows).toHaveLength(1);
 
-        await expect(durable().getBySubId(expired.row.subId)).resolves.toBeNull();
+        await expect(
+            durable().getBySubId(expired.row.subId),
+        ).resolves.toBeNull();
         await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
             live.row,
         ]);
@@ -754,19 +873,42 @@ describe('the expiry sweep', () => {
 
     it('leaves a subscription with no expiry alone', async () => {
         await durable().create(input());
-        await expect(durable().sweepExpired(500)).resolves.toBe(0);
+        expect((await durable().sweepExpired(500)).rows).toHaveLength(0);
         await expect(durable().countForHolder(userId)).resolves.toMatchObject({
             total: 1,
         });
     });
 });
 
+describe('a sweep reading a lagging replica', () => {
+    it('keeps a row the primary resumed since, and does not report it', async () => {
+        const { row } = await durable().create(input());
+        await durable().suspend([row], 'permission_revoked');
+        const [stale] = await env.server.clients.db.pread(
+            'SELECT * FROM `event_subscriptions` WHERE `sub_id` = ?',
+            [row.subId],
+        );
+        await durable().resume([row]);
+        // The replica still has it suspended.
+        vi.spyOn(env.server.clients.db, 'read').mockResolvedValueOnce([stale]);
+
+        const reaped = await durable().sweepSuspended(
+            Math.floor(Date.now() / 1000) + 3600,
+            500,
+        );
+
+        expect(reaped.rows).toEqual([]);
+        await expect(durable().getBySubId(row.subId)).resolves.toMatchObject({
+            subId: row.subId,
+            suspendedAt: null,
+        });
+    });
+});
+
 describe('listHolderIdsForApp', () => {
     it('names every distinct holder bound to the app`s handlers', async () => {
         const appUid = `app-${uuidv4()}`;
-        await durable().create(
-            input({ appUid, holderUserId: userId }),
-        );
+        await durable().create(input({ appUid, holderUserId: userId }));
         await durable().create(
             input({
                 appUid,
@@ -774,9 +916,7 @@ describe('listHolderIdsForApp', () => {
             }),
         );
         // A second row for the same holder must not double the name.
-        await durable().create(
-            input({ appUid, holderUserId: userId }),
-        );
+        await durable().create(input({ appUid, holderUserId: userId }));
 
         const holders = await durable().listHolderIdsForApp(appUid);
         expect(holders.sort()).toEqual([userId, otherUserId].sort());
@@ -784,48 +924,44 @@ describe('listHolderIdsForApp', () => {
 
     it('finds a suspended row`s holder too', async () => {
         const appUid = `app-${uuidv4()}`;
-        const { row } = await durable().create(
-            input({ appUid }),
-        );
+        const { row } = await durable().create(input({ appUid }));
         await durable().suspend([row], 'handler_not_found');
 
         expect(await durable().listHolderIdsForApp(appUid)).toEqual([userId]);
     });
 
     it('answers an empty list for an app with nothing bound', async () => {
-        expect(
-            await durable().listHolderIdsForApp(`app-${uuidv4()}`),
-        ).toEqual([]);
+        expect(await durable().listHolderIdsForApp(`app-${uuidv4()}`)).toEqual(
+            [],
+        );
     });
 });
 
 describe('reapForApp', () => {
     it('deletes every row bound to the app and bumps its owner`s generation', async () => {
         const appUid = `app-${uuidv4()}`;
-        const mine = await durable().create(
-            input({ appUid }),
-        );
+        const mine = await durable().create(input({ appUid }));
         const untouched = await durable().create(input());
         const before = await cache().getGeneration(userId);
 
-        await expect(durable().reapForApp(appUid, 500)).resolves.toBe(1);
+        const reaped = await durable().reapForApp(appUid, 500);
+        expect(reaped.rows.map((row) => row.subId)).toEqual([mine.row.subId]);
 
         await expect(durable().getBySubId(mine.row.subId)).resolves.toBeNull();
         await expect(
             durable().getBySubId(untouched.row.subId),
         ).resolves.not.toBeNull();
-        await expect(cache().getGeneration(userId)).resolves.toBeGreaterThan(
-            before,
-        );
+        const after = await cache().getGeneration(userId);
+        expect(after).toBeGreaterThan(before);
+        // The bump the caller publishes to the other regions.
+        expect(reaped.bumps).toEqual([{ userId, generation: after }]);
     });
 
     it('never touches another app`s rows', async () => {
         const appUid = `app-${uuidv4()}`;
         const otherAppUid = `app-${uuidv4()}`;
         await durable().create(input({ appUid }));
-        const theirs = await durable().create(
-            input({ appUid: otherAppUid }),
-        );
+        const theirs = await durable().create(input({ appUid: otherAppUid }));
 
         await durable().reapForApp(appUid, 500);
 
@@ -837,7 +973,7 @@ describe('reapForApp', () => {
     it('answers 0 for an app with nothing to reap', async () => {
         await expect(
             durable().reapForApp(`app-${uuidv4()}`, 500),
-        ).resolves.toBe(0);
+        ).resolves.toEqual({ rows: [], bumps: [] });
     });
 });
 
@@ -904,3 +1040,298 @@ describe('suspending', () => {
         expect(second.bumps).toEqual([]);
     });
 });
+
+describe('rows a schedule owns', () => {
+    const scheduleInput = (
+        over: Partial<DurableSubscriptionInput> = {},
+    ): DurableSubscriptionInput => {
+        const uid = uuidv4();
+        return input({
+            subject: scheduleToken(uid),
+            token: scheduleToken(uid),
+            anchorUid: uid,
+            anchorPath: '',
+            appUid: 'app-scheduler',
+            delivery: 'single',
+            handlerName: 'tick',
+            permission: 'events:background',
+            ...over,
+        });
+    };
+
+    it('are created past the durable quota, uncached and with no bump', async () => {
+        const limits = { perUser: 1, perApp: 1 };
+        await durable().create(input({ limits }));
+        const before = await cache().getGeneration(userId);
+
+        const created = await durable().create(scheduleInput({ limits }));
+
+        expect(created.bump).toBeNull();
+        await expect(cache().getGeneration(userId)).resolves.toBe(before);
+        await expect(
+            cache().getForTokens(userId, [created.row.token]),
+        ).resolves.toEqual([]);
+        await expect(
+            durable().getBySubId(created.row.subId),
+        ).resolves.toMatchObject({ token: created.row.token });
+    });
+
+    it('never enter the dispatch cache or what a cold region rebuilds', async () => {
+        const { row } = await durable().create(scheduleInput());
+        await expect(
+            durable().listDeliverableForOwner(userId),
+        ).resolves.toEqual([]);
+
+        await cache().markRegionCold(userId);
+        await durable().warmRegion(userId);
+        await expect(cache().userHasAny(userId)).resolves.toBe(false);
+
+        await durable().suspend([row], 'no_credit');
+        await durable().resume([
+            { ...row, suspendedAt: 1, suspendedReason: 'no_credit' },
+        ]);
+        await expect(
+            cache().getForTokens(userId, [row.token]),
+        ).resolves.toEqual([]);
+        await expect(cache().userHasAny(userId)).resolves.toBe(false);
+    });
+
+    it('bump no generation when suspended or resumed', async () => {
+        const { row } = await durable().create(scheduleInput());
+        const before = await cache().getGeneration(userId);
+
+        const { suspended, bumps } = await durable().suspend([row], 'failures');
+        expect(suspended).toHaveLength(1);
+        expect(bumps).toEqual([]);
+        await expect(durable().resume(suspended)).resolves.toEqual([]);
+        await expect(cache().getGeneration(userId)).resolves.toBe(before);
+    });
+
+    it('count toward no durable quota, and are not in the durable listing', async () => {
+        await durable().create(input());
+        await durable().create(scheduleInput());
+
+        await expect(durable().countForHolder(userId)).resolves.toEqual({
+            total: 1,
+            forApp: 0,
+            terminal: 0,
+        });
+        const listed = await durable().listForHolder(userId, {
+            includeTotal: true,
+        });
+        expect(listed.total).toBe(1);
+        expect(listed.items.map((row) => row.token)).toEqual([token()]);
+    });
+
+    it('are kept when the oldest revoked rows are trimmed', async () => {
+        const base = Math.floor(Date.now() / 1000) - 10_000;
+        const { row: scheduled } = await durable().create(scheduleInput());
+        await env.server.clients.db.write(
+            'UPDATE `event_subscriptions` SET `suspended_at` = ?, ' +
+                '`suspended_reason` = ? WHERE `sub_id` = ?',
+            [base - 1, 'permission_revoked', scheduled.subId],
+        );
+        for (let i = 0; i < EVENTS_REVOKED_SUBSCRIPTIONS_KEPT + 1; i++) {
+            const { row } = await durable().create(input());
+            await env.server.clients.db.write(
+                'UPDATE `event_subscriptions` SET `suspended_at` = ?, ' +
+                    '`suspended_reason` = ? WHERE `sub_id` = ?',
+                [base + i, 'permission_revoked', row.subId],
+            );
+        }
+
+        await durable().create(input());
+
+        await expect(
+            durable().getBySubId(scheduled.subId),
+        ).resolves.not.toBeNull();
+        await expect(durable().countForHolder(userId)).resolves.toMatchObject({
+            terminal: EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
+        });
+    });
+
+    it('are read, a page at a time, by what settles a holder', async () => {
+        const made: string[] = [];
+        for (let i = 0; i < 3; i++)
+            made.push((await durable().create(scheduleInput())).row.subId);
+        made.push((await durable().create(input())).row.subId);
+
+        const first = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            0,
+            2,
+        );
+        expect(first.rows.map((row) => row.subId)).toEqual(made.slice(0, 2));
+        expect(first.nextId).not.toBeNull();
+        const second = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            first.nextId!,
+            2,
+        );
+        expect(second.rows.map((row) => row.subId)).toEqual(made.slice(2));
+        // A full page always says to look again; the next one is empty.
+        const third = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            second.nextId ?? Number.MAX_SAFE_INTEGER,
+            2,
+        );
+        expect(third).toEqual({ rows: [], nextId: null });
+    });
+
+    it('are read, a page at a time, by what a top-up resumes', async () => {
+        const rows = [];
+        for (let i = 0; i < 3; i++)
+            rows.push((await durable().create(scheduleInput())).row);
+        await durable().suspend(rows, 'no_credit');
+
+        const first = await durable().listSuspendedForHolderPage(
+            userId,
+            'no_credit',
+            0,
+            2,
+        );
+        expect(first.rows).toHaveLength(2);
+        const second = await durable().listSuspendedForHolderPage(
+            userId,
+            'no_credit',
+            first.nextId!,
+            2,
+        );
+        expect(second.rows.map((row) => row.subId)).toEqual([rows[2].subId]);
+        expect(second.nextId).toBeNull();
+    });
+
+    it('are found by handler and by app, like any other row', async () => {
+        const { row } = await durable().create(scheduleInput());
+        await expect(
+            durable().listByHandler('app-scheduler', 'app', 0, 'tick'),
+        ).resolves.toEqual([expect.objectContaining({ subId: row.subId })]);
+        await expect(
+            durable().listHolderIdsForApp('app-scheduler'),
+        ).resolves.toEqual([userId]);
+    });
+
+    it('come back from a reap without touching a backlog', async () => {
+        const { row } = await durable().create(scheduleInput());
+        const regular = await durable().create(
+            input({ appUid: 'app-scheduler' }),
+        );
+        const purge = vi.spyOn(env.server.stores.pendingDelivery, 'purge');
+        try {
+            const reaped = await durable().reapForApp('app-scheduler', 500);
+            expect(reaped.rows.map((r) => r.subId).sort()).toEqual(
+                [row.subId, regular.row.subId].sort(),
+            );
+            expect(purge.mock.calls.map(([subId]) => subId)).toEqual([
+                regular.row.subId,
+            ]);
+            // Only the cached row's owner was bumped.
+            expect(reaped.bumps).toHaveLength(1);
+        } finally {
+            purge.mockRestore();
+        }
+    });
+
+    it('are deleted by their schedule`s id, whether or not it recorded the row', async () => {
+        const uid = uuidv4();
+        await durable().create(
+            scheduleInput({
+                subject: scheduleToken(uid),
+                token: scheduleToken(uid),
+                anchorUid: uid,
+            }),
+        );
+        await expect(durable().removeScheduleSubscription(uid)).resolves.toBe(
+            true,
+        );
+        await expect(durable().removeScheduleSubscription(uid)).resolves.toBe(
+            false,
+        );
+    });
+});
+
+describe('statements a schedule store composes', () => {
+    const createScheduleRow = async () => {
+        const uid = uuidv4();
+        const { row } = await durable().create(
+            input({
+                subject: scheduleToken(uid),
+                token: scheduleToken(uid),
+                anchorUid: uid,
+                anchorPath: '',
+                appUid: 'app-scheduler',
+                delivery: 'single',
+                handlerName: 'tick',
+                context: '{"region":"eu"}',
+                permission: 'events:background',
+            }),
+        );
+        return { uid, row };
+    };
+
+    it('delete a schedule`s row only while the guard holds', async () => {
+        const { uid, row } = await createScheduleRow();
+        const db = env.server.clients.db;
+
+        await db.batchWrite([
+            durable().scheduleSubscriptionDeleteEntry(uid, {
+                table: 'user',
+                where: '`id` = ?',
+                values: [-1],
+            }),
+        ]);
+        await expect(durable().getBySubId(row.subId)).resolves.not.toBeNull();
+
+        await db.batchWrite([
+            durable().scheduleSubscriptionDeleteEntry(uid, {
+                table: 'user',
+                where: '`id` = ?',
+                values: [userId],
+            }),
+        ]);
+        await expect(durable().getBySubId(row.subId)).resolves.toBeNull();
+    });
+
+    it('join a schedule`s row under the sub_* columns', async () => {
+        const { row } = await createScheduleRow();
+        const join = durable().scheduleSubscriptionJoin('sub', 'x.`sub_id`');
+        const read = (subId: string) =>
+            env.server.clients.db.pread(
+                `SELECT ${join.columns} FROM (SELECT ? AS \`sub_id\`) x ${join.join}`,
+                [subId],
+            );
+
+        const [found] = await read(row.subId);
+        expect(found).toMatchObject({
+            sub_suspended_at: null,
+            sub_suspended_reason: null,
+            sub_handler_name: 'tick',
+        });
+        expect(Number(found.sub_row_id)).toBeGreaterThan(0);
+        expect(found.sub_context).toBe('{"region":"eu"}');
+
+        const [missing] = await read('no-such-sub');
+        expect(missing).toMatchObject({
+            sub_row_id: null,
+            sub_handler_name: null,
+        });
+    });
+
+    it('tell a schedule whose row is gone', async () => {
+        const { uid, row } = await createScheduleRow();
+        const absent = async (subId: string) => {
+            const [result] = await env.server.clients.db.pread(
+                `SELECT CASE WHEN ${durable().scheduleSubscriptionAbsent('?')} THEN 1 ELSE 0 END AS \`absent\``,
+                [subId],
+            );
+            return Number(result.absent);
+        };
+
+        await expect(absent(row.subId)).resolves.toBe(0);
+        await durable().removeScheduleSubscription(uid);
+        await expect(absent(row.subId)).resolves.toBe(1);
+    });
+});

src/backend/stores/events/DurableSubscriptionStore.ts

@@ -23,7 +23,9 @@ import {
     EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
     type TieredQuota,
 } from '../../controllers/events/limits.js';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
 import { HttpError } from '../../core/http/HttpError.js';
+import type { SqlGuard } from '../../core/schedules.js';
 import type { DeliveryClass } from '../../services/events/registry.js';
 import type { FsOp } from '../../services/events/subjects.js';
 import {
@@ -37,7 +39,14 @@ import {
     EVENT_SUBSCRIPTION_WIDTHS,
 } from './columnWidths.js';
 import type { GenerationBump } from './EventSubscriptionStore.js';
-import type { DurableSubscription, SubscriptionPermission } from './types.js';
+import {
+    isScheduleRow,
+    SCHEDULE_TOKEN_PREFIX,
+    scheduleToken,
+    type DurableSubscription,
+    type HandlerScopeKind,
+    type SubscriptionPermission,
+} from './types.js';
 
 /**
  * Subscriptions that outlive the connection that made them.
@@ -66,6 +75,9 @@ export const DURABLE_CONTEXT_MAX_BYTES = 4096;
 export const DURABLE_LIST_DEFAULT_LIMIT = 50;
 export const DURABLE_LIST_LIMIT_CAP = 200;
 
+/** Rows one page of a holder-wide settle takes. */
+export const HOLDER_SETTLE_BATCH = 500;
+
 /**
  * Rows one handler-lifecycle pass takes. A removal suspends its dependents in
  * batches so a widely-used name cannot make one call hold the whole set.
@@ -89,6 +101,8 @@ export interface DurableSubscriptionInput {
     op: FsOp | null;
     delivery: DeliveryClass;
     handlerName: string | null;
+    /** Which set `handlerName` refers to. Omitted means the app's. */
+    handlerScope?: HandlerScopeKind;
     context: string | null;
     permission: SubscriptionPermission;
     expiresAt: number | null;
@@ -101,6 +115,15 @@ export interface DurableSubscriptionInput {
     limits?: TieredQuota;
 }
 
+/**
+ * What a reap deleted, and the generation it bumped per owner. The caller
+ * publishes the bumps, since the reap only invalidates this region.
+ */
+export interface DurableReap {
+    rows: DurableSubscription[];
+    bumps: GenerationBump[];
+}
+
 /** One keyset page of rows, and where the next page starts. */
 export interface DurablePage {
     rows: DurableSubscription[];
@@ -148,6 +171,13 @@ const TERMINAL_CLAUSE =
     '(`suspended_at` IS NOT NULL AND `suspended_reason` IN (' +
     `${TERMINAL_SUSPENDED_REASONS.map(() => '?').join(', ')}))`;
 
+/**
+ * Leaves out the rows schedules own: they have their own caps and listing, and
+ * never enter the dispatch cache. Binds {@link SCHEDULE_TOKEN_PATTERN}.
+ */
+const NOT_SCHEDULE_CLAUSE = '`token` NOT LIKE ?';
+const SCHEDULE_TOKEN_PATTERN = `${SCHEDULE_TOKEN_PREFIX}%`;
+
 /** Where a row is moving to when its anchor is deleted under it. */
 export interface ReanchorInput {
     token: string;
@@ -167,6 +197,12 @@ const contextTooLarge = (): HttpError =>
         { legacyCode: 'events_context_too_large' },
     );
 
+export const assertDurableContext = (context: string | null): void => {
+    if (context === null) return;
+    if (Buffer.byteLength(context, 'utf8') > DURABLE_CONTEXT_MAX_BYTES)
+        throw contextTooLarge();
+};
+
 const quotaReached = (limit: number, scope: 'account' | 'app'): HttpError =>
     new HttpError(
         429,
@@ -207,6 +243,16 @@ const quotaRefusal = (
     return null;
 };
 
+/**
+ * A created row; `reaped` holds the bumps of terminal rows it trimmed. A
+ * schedule's row is never cached, so it comes back with no bump.
+ */
+export interface DurableCreated {
+    row: DurableSubscription;
+    bump: GenerationBump | null;
+    reaped: GenerationBump[];
+}
+
 /**
  * The quota refusal for a create that concurrent creates beat to the last slot
  * after its own insert. Its row is already gone again; `bump` still has to be
@@ -264,6 +310,9 @@ const toRow = (row: Record<string, unknown>): DurableSubscription => ({
         row.handler_name === null || row.handler_name === undefined
             ? null
             : String(row.handler_name),
+    ...(Number(row.handler_scope) === 1
+        ? { handlerScope: 'user' as const }
+        : {}),
     context:
         row.context === null || row.context === undefined
             ? null
@@ -281,7 +330,7 @@ const toRow = (row: Record<string, unknown>): DurableSubscription => ({
 const SELECT_COLUMNS =
     '`id`, `sub_id`, `token`, `owner_user_id`, `holder_user_id`, `app_uid`, ' +
     '`subject`, `anchor_uid`, `anchor_path`, `match`, `delivery`, `ops`, ' +
-    '`handler_name`, `context`, `permission`, `include_value`, ' +
+    '`handler_name`, `handler_scope`, `context`, `permission`, `include_value`, ' +
     '`suspended_at`, `suspended_reason`, `expires_at`, `created_at`';
 
 export class DurableSubscriptionStore extends PuterStore {
@@ -291,11 +340,13 @@ export class DurableSubscriptionStore extends PuterStore {
      * Insert one subscription and put it in this region's cache before
      * returning. The generation bump is what tells every other region, and the
      * caller broadcasts it.
+     *
+     * A schedule's row skips the durable quotas and the cache, so it comes back
+     * with no bump: its caps are the schedule store's.
      */
-    async create(
-        input: DurableSubscriptionInput,
-    ): Promise<{ row: DurableSubscription; bump: GenerationBump }> {
-        this.#assertContext(input.context);
+    async create(input: DurableSubscriptionInput): Promise<DurableCreated> {
+        assertDurableContext(input.context);
+        const forSchedule = isScheduleRow(input);
         assertColumnWidths(EVENT_SUBSCRIPTION_WIDTHS, {
             token: input.token,
             appUid: input.appUid,
@@ -307,12 +358,14 @@ export class DurableSubscriptionStore extends PuterStore {
             permission: input.permission,
         });
 
-        const refusal = quotaRefusal(
-            input,
-            await this.countForHolder(input.holderUserId, input.appUid),
-            0,
-        );
-        if (refusal) throw refusal;
+        if (!forSchedule) {
+            const refusal = quotaRefusal(
+                input,
+                await this.countForHolder(input.holderUserId, input.appUid),
+                0,
+            );
+            if (refusal) throw refusal;
+        }
 
         const row: DurableSubscription = {
             durable: true,
@@ -329,6 +382,9 @@ export class DurableSubscriptionStore extends PuterStore {
             permission: input.permission,
             delivery: input.delivery,
             handlerName: input.handlerName,
+            ...(input.handlerScope === 'user'
+                ? { handlerScope: 'user' as const }
+                : {}),
             context: input.context,
             ...(input.includeValue ? { includeValue: true } : {}),
             expiresAt: input.expiresAt,
@@ -351,6 +407,7 @@ export class DurableSubscriptionStore extends PuterStore {
             delivery: row.delivery,
             ops: row.op,
             handler_name: row.handlerName,
+            handler_scope: row.handlerScope === 'user' ? 1 : 0,
             targets: legacyTargets(row.appUid),
             context: row.context,
             permission: row.permission,
@@ -358,6 +415,7 @@ export class DurableSubscriptionStore extends PuterStore {
             expires_at: row.expiresAt,
             created_at: row.createdAt,
         });
+        if (forSchedule) return { row, bump: null, reaped: [] };
 
         // Concurrent creates can all pass the count above. Counted again with
         // this row in it, each one past the cap takes itself back out — at
@@ -368,18 +426,20 @@ export class DurableSubscriptionStore extends PuterStore {
         );
         const lost = quotaRefusal(input, held, 1);
         if (lost) throw new DurableQuotaRaceLost(lost, await this.remove(row));
-        if (held.terminal > EVENTS_REVOKED_SUBSCRIPTIONS_KEPT)
-            await this.#dropOldestTerminal(
-                input.holderUserId,
-                held.terminal - EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
-            );
+        const reaped =
+            held.terminal > EVENTS_REVOKED_SUBSCRIPTIONS_KEPT
+                ? await this.#dropOldestTerminal(
+                      input.holderUserId,
+                      held.terminal - EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
+                  )
+                : [];
 
         // Write-through, and over the whole owner rather than the one row: it
         // costs the same indexed read as a warm would, on a path rate-limited
         // to a few calls a minute, and it leaves this region needing nothing
         // from the table before it can deliver.
         await this.#rebuildRegion(row.ownerUserId);
-        return { row, bump: await this.#bump(row.ownerUserId) };
+        return { row, bump: await this.#bump(row.ownerUserId), reaped };
     }
 
     /**
@@ -439,8 +499,9 @@ export class DurableSubscriptionStore extends PuterStore {
                 });
         }
 
-        await this.stores.eventSubscription.dropDurables(suspended);
-        const owners = new Set(suspended.map((row) => row.ownerUserId));
+        const cached = suspended.filter((row) => !isScheduleRow(row));
+        await this.stores.eventSubscription.dropDurables(cached);
+        const owners = new Set(cached.map((row) => row.ownerUserId));
         const bumps = await Promise.all(
             [...owners].map((owner) => this.#bump(owner)),
         );
@@ -468,6 +529,7 @@ export class DurableSubscriptionStore extends PuterStore {
         // owner from the first row it is given.
         const byOwner = new Map<number, DurableSubscription[]>();
         for (const row of rows) {
+            if (isScheduleRow(row)) continue;
             const live: DurableSubscription = {
                 ...row,
                 suspendedAt: null,
@@ -541,7 +603,7 @@ export class DurableSubscriptionStore extends PuterStore {
         row: DurableSubscription,
         context: string | null,
     ): Promise<{ row: DurableSubscription; bump: GenerationBump } | null> {
-        this.#assertContext(context);
+        assertDurableContext(context);
         const written = await this.clients.db.write(
             `UPDATE \`${TABLE}\` SET \`context\` = ? WHERE \`sub_id\` = ?`,
             [context, row.subId],
@@ -573,8 +635,12 @@ export class DurableSubscriptionStore extends PuterStore {
      * Reap rows past their expiry, dropping each from the cache so the region
      * stops delivering against it without waiting for a rebuild.
      */
-    async sweepExpired(batchSize: number): Promise<number> {
-        return this.#reap(await this.#listExpired(nowSeconds(), batchSize));
+    async sweepExpired(batchSize: number): Promise<DurableReap> {
+        const cutoff = nowSeconds();
+        return this.#reap(await this.#listExpired(cutoff, batchSize), {
+            where: '`expires_at` IS NOT NULL AND `expires_at` <= ?',
+            values: [cutoff],
+        });
     }
 
     /**
@@ -582,8 +648,14 @@ export class DurableSubscriptionStore extends PuterStore {
      * never resumes is a row kept only so its holder can see why it stopped,
      * and that answer has a shelf life.
      */
-    async sweepSuspended(cutoff: number, batchSize: number): Promise<number> {
-        return this.#reap(await this.#listSuspendedBefore(cutoff, batchSize));
+    async sweepSuspended(
+        cutoff: number,
+        batchSize: number,
+    ): Promise<DurableReap> {
+        return this.#reap(await this.#listSuspendedBefore(cutoff, batchSize), {
+            where: '`suspended_at` IS NOT NULL AND `suspended_at` <= ?',
+            values: [cutoff],
+        });
     }
 
     // -- Reads -------------------------------------------------------
@@ -622,8 +694,16 @@ export class DurableSubscriptionStore extends PuterStore {
 
         // The scope half of the predicate is what the total counts over; the
         // cursor half only positions one page inside it.
-        const scope = ['`holder_user_id` = ?', this.#unexpiredClause()];
-        const scopeParams: unknown[] = [holderUserId, nowSeconds()];
+        const scope = [
+            '`holder_user_id` = ?',
+            this.#unexpiredClause(),
+            NOT_SCHEDULE_CLAUSE,
+        ];
+        const scopeParams: unknown[] = [
+            holderUserId,
+            nowSeconds(),
+            SCHEDULE_TOKEN_PATTERN,
+        ];
         if (options.appUid !== undefined && options.appUid !== null) {
             scope.push('`app_uid` = ?');
             scopeParams.push(options.appUid);
@@ -675,7 +755,8 @@ export class DurableSubscriptionStore extends PuterStore {
                 `SUM(CASE WHEN ${TERMINAL_CLAUSE} THEN 0 ` +
                 'WHEN `app_uid` = ? THEN 1 ELSE 0 END) AS `for_app`, ' +
                 `SUM(CASE WHEN ${TERMINAL_CLAUSE} THEN 1 ELSE 0 END) AS \`terminal\` ` +
-                `FROM \`${TABLE}\` WHERE \`holder_user_id\` = ? AND ${this.#unexpiredClause()}`,
+                `FROM \`${TABLE}\` WHERE \`holder_user_id\` = ? AND ${this.#unexpiredClause()} ` +
+                `AND ${NOT_SCHEDULE_CLAUSE}`,
             [
                 ...terminal,
                 ...terminal,
@@ -683,6 +764,7 @@ export class DurableSubscriptionStore extends PuterStore {
                 ...terminal,
                 holderUserId,
                 nowSeconds(),
+                SCHEDULE_TOKEN_PATTERN,
             ],
         );
         return {
@@ -693,22 +775,26 @@ export class DurableSubscriptionStore extends PuterStore {
     }
 
     /**
-     * The holder's live rows, which is what a revocation has to consider. Same
-     * index the listing and the quota use — passing `appUid` narrows to one
-     * app's rows, which is what a grant made to that app can have authorized.
+     * The holder's live rows, which is what a revocation has to consider, one
+     * keyset page at a time. Same index the listing and the quota use — passing
+     * `appUid` narrows to one app's rows, which is what a grant made to that
+     * app can have authorized. Schedule rows are included: a holder can hold
+     * far more of them than the durable quota allows, which is why this pages.
      *
      * `includeSuspended` also returns rows out of service for a reason that
      * lifts. A revocation needs them: those reasons hold a backlog, and a row
      * skipped here comes back in service on the next resume and hands it over.
+     * Rows suspended for good are left out.
      *
-     * Bounded by the per-account quota, so the whole set fits one read. Rows
-     * suspended for good hold no slot, so they are left out.
+     * `excludeSchedules` is for a settle no schedule row can match.
      */
-    async listActiveForHolder(
+    async listActiveForHolderPage(
         holderUserId: number,
         appUid: string | null,
-        opts: { includeSuspended?: boolean } = {},
-    ): Promise<DurableSubscription[]> {
+        afterId: number,
+        batchSize: number = HOLDER_SETTLE_BATCH,
+        opts: { includeSuspended?: boolean; excludeSchedules?: boolean } = {},
+    ): Promise<DurablePage> {
         const where = [
             '`holder_user_id` = ?',
             opts.includeSuspended
@@ -721,21 +807,25 @@ export class DurableSubscriptionStore extends PuterStore {
             ...(opts.includeSuspended ? TERMINAL_SUSPENDED_REASONS : []),
             nowSeconds(),
         ];
+        if (opts.excludeSchedules) {
+            where.push(NOT_SCHEDULE_CLAUSE);
+            params.push(SCHEDULE_TOKEN_PATTERN);
+        }
         if (appUid !== null) {
             where.push('`app_uid` = ?');
             params.push(appUid);
         }
-        const rows = await this.clients.db.pread(
-            `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
-                `WHERE ${where.join(' AND ')} ORDER BY \`id\` LIMIT ?`,
-            [...params, EVENTS_DURABLE_SUBSCRIPTIONS_MAX],
-        );
-        return rows.map(toRow);
+        return this.#page(where, params, afterId, batchSize, {
+            primary: true,
+        });
     }
 
     /**
-     * The rows bound to one of an app's handler names. What a removal has to
+     * The rows bound to one of a scope's handler names. What a removal has to
      * suspend, and — asking for the suspended half — what a republish resumes.
+     * `holder` is read for user scope only. App scope never returns user-scope
+     * rows; user scope is read through the holder index, so it never walks the
+     * other holders of a popular app.
      *
      * Bounded by the per-account quota times nothing: a widely-used handler can
      * carry more rows than one read should return, so this is a page and the
@@ -743,11 +833,28 @@ export class DurableSubscriptionStore extends PuterStore {
      */
     async listByHandler(
         appUid: string,
+        scope: HandlerScopeKind,
+        holder: number,
         handlerName: string,
         options: { suspendedReason?: SuspendedReason; limit?: number } = {},
     ): Promise<DurableSubscription[]> {
-        const where = ['`app_uid` = ?', '`handler_name` = ?'];
-        const params: unknown[] = [appUid, handlerName];
+        const where =
+            scope === 'user'
+                ? [
+                      '`holder_user_id` = ?',
+                      '`app_uid` = ?',
+                      '`handler_scope` = 1',
+                      '`handler_name` = ?',
+                  ]
+                : [
+                      '`app_uid` = ?',
+                      '`handler_scope` = 0',
+                      '`handler_name` = ?',
+                  ];
+        const params: unknown[] =
+            scope === 'user'
+                ? [holder, appUid, handlerName]
+                : [appUid, handlerName];
         if (options.suspendedReason === undefined) {
             where.push('`suspended_at` IS NULL');
         } else {
@@ -768,25 +875,27 @@ export class DurableSubscriptionStore extends PuterStore {
 
     /**
      * One holder's rows suspended for a given reason — what a resume condition
-     * that belongs to the account rather than to a handler releases.
+     * that belongs to the account rather than to a handler releases — one
+     * keyset page at a time.
      */
-    async listSuspendedForHolder(
+    async listSuspendedForHolderPage(
         holderUserId: number,
         reason: SuspendedReason,
-    ): Promise<DurableSubscription[]> {
-        const rows = await this.clients.db.pread(
-            `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
-                'WHERE `holder_user_id` = ? AND `suspended_at` IS NOT NULL ' +
-                `AND \`suspended_reason\` = ? AND ${this.#unexpiredClause()} ` +
-                'ORDER BY `id` LIMIT ?',
+        afterId: number,
+        batchSize: number = HOLDER_SETTLE_BATCH,
+    ): Promise<DurablePage> {
+        return this.#page(
             [
-                holderUserId,
-                reason,
-                nowSeconds(),
-                EVENTS_DURABLE_SUBSCRIPTIONS_MAX,
+                '`holder_user_id` = ?',
+                '`suspended_at` IS NOT NULL',
+                '`suspended_reason` = ?',
+                this.#unexpiredClause(),
             ],
+            [holderUserId, reason, nowSeconds()],
+            afterId,
+            batchSize,
+            { primary: true },
         );
-        return rows.map(toRow);
     }
 
     /**
@@ -819,8 +928,8 @@ export class DurableSubscriptionStore extends PuterStore {
         const rows = await this.clients.db.pread(
             `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
                 'WHERE `owner_user_id` = ? AND `suspended_at` IS NULL AND ' +
-                `${this.#unexpiredClause()}`,
-            [ownerUserId, nowSeconds()],
+                `${this.#unexpiredClause()} AND ${NOT_SCHEDULE_CLAUSE}`,
+            [ownerUserId, nowSeconds(), SCHEDULE_TOKEN_PATTERN],
         );
         return rows.map(toRow);
     }
@@ -846,30 +955,98 @@ export class DurableSubscriptionStore extends PuterStore {
      * What app deletion reaps: `#reap` already purges the backlog, drops the
      * cache and bumps a generation per owner, so this is the whole of it.
      */
-    async reapForApp(appUid: string, batchSize: number): Promise<number> {
+    async reapForApp(appUid: string, batchSize: number): Promise<DurableReap> {
         return this.#reap(await this.#listForApp(appUid, batchSize));
     }
 
+    /**
+     * Delete the row a schedule owns, whether or not the schedule ever recorded
+     * its id: a create that crashed between the two inserts leaves one it
+     * didn't. Never cached, so there is nothing to drop or bump. Called by
+     * extensions.
+     */
+    async removeScheduleSubscription(scheduleUid: string): Promise<boolean> {
+        const written = await this.clients.db.write(
+            `DELETE FROM \`${TABLE}\` WHERE \`token\` = ?`,
+            [scheduleToken(scheduleUid)],
+        );
+        return written.anyRowsAffected;
+    }
+
+    // Statements and fragments for the schedules' own transactions and reads,
+    // so no SQL outside this store names the table. Called by extensions.
+
+    /**
+     * Delete the row a schedule owns, only while `guard` holds: for a batch
+     * that ends the schedule in the same transaction.
+     */
+    scheduleSubscriptionDeleteEntry(
+        scheduleUid: string,
+        guard: SqlGuard,
+    ): BatchEntry {
+        return {
+            statement:
+                `DELETE FROM \`${TABLE}\` WHERE \`token\` = ? ` +
+                `AND EXISTS (SELECT 1 FROM \`${guard.table}\` WHERE ${guard.where})`,
+            values: [scheduleToken(scheduleUid), ...guard.values],
+        };
+    }
+
+    /**
+     * A `LEFT JOIN` of the row a schedule owns, as `alias`, on `subIdColumn`,
+     * and the columns it selects: `sub_row_id`, `sub_suspended_at`,
+     * `sub_suspended_reason`, `sub_handler_name`, `sub_handler_scope` and
+     * `sub_context`.
+     */
+    scheduleSubscriptionJoin(
+        alias: string,
+        subIdColumn: string,
+    ): { columns: string; join: string } {
+        return {
+            columns:
+                `${alias}.\`id\` AS \`sub_row_id\`, ` +
+                `${alias}.\`suspended_at\` AS \`sub_suspended_at\`, ` +
+                `${alias}.\`suspended_reason\` AS \`sub_suspended_reason\`, ` +
+                `${alias}.\`handler_name\` AS \`sub_handler_name\`, ` +
+                `${alias}.\`handler_scope\` AS \`sub_handler_scope\`, ` +
+                `${alias}.\`context\` AS \`sub_context\``,
+            join: `LEFT JOIN \`${TABLE}\` ${alias} ON ${alias}.\`sub_id\` = ${subIdColumn}`,
+        };
+    }
+
+    /** A condition true when no subscription row has the id `subIdExpr`. */
+    scheduleSubscriptionAbsent(subIdExpr: string): string {
+        return `NOT EXISTS (SELECT 1 FROM \`${TABLE}\` e WHERE e.\`sub_id\` = ${subIdExpr})`;
+    }
+
     // -- Internals ---------------------------------------------------
 
     /**
-     * Delete a holder's oldest rows suspended for good, past what `list` keeps.
-     * Best-effort: the next subscribe tries again.
+     * Delete a holder's oldest rows suspended for good, past what `list` keeps,
+     * and return the reap's bumps. Best-effort: the next subscribe tries
+     * again.
      */
     async #dropOldestTerminal(
         holderUserId: number,
         count: number,
-    ): Promise<void> {
+    ): Promise<GenerationBump[]> {
         try {
             const rows = await this.clients.db.pread(
                 `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
                     `WHERE \`holder_user_id\` = ? AND ${TERMINAL_CLAUSE} ` +
+                    `AND ${NOT_SCHEDULE_CLAUSE} ` +
                     'ORDER BY `suspended_at`, `id` LIMIT ?',
-                [holderUserId, ...TERMINAL_SUSPENDED_REASONS, count],
+                [
+                    holderUserId,
+                    ...TERMINAL_SUSPENDED_REASONS,
+                    SCHEDULE_TOKEN_PATTERN,
+                    count,
+                ],
             );
-            await this.#reap(rows.map(toRow));
+            return (await this.#reap(rows.map(toRow))).bumps;
         } catch (err) {
             console.warn('[events] could not drop old revoked rows', err);
+            return [];
         }
     }
 
@@ -891,15 +1068,23 @@ export class DurableSubscriptionStore extends PuterStore {
         return rows.map(toRow);
     }
 
-    /** One page of a whole-table scan, ordered and positioned by primary key. */
+    /**
+     * One keyset page, ordered and positioned by primary key. The replica
+     * unless `primary`, which settles need: a lagging read would skip a row
+     * just created.
+     */
     async #page(
         where: string[],
         params: unknown[],
         afterId: number,
         batchSize: number,
+        opts: { primary?: boolean } = {},
     ): Promise<DurablePage> {
         const limit = Math.max(1, Math.floor(batchSize));
-        const rows = await this.clients.db.read(
+        const read = opts.primary
+            ? this.clients.db.pread.bind(this.clients.db)
+            : this.clients.db.read.bind(this.clients.db);
+        const rows = await read(
             `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
                 `WHERE ${[...where, '`id` > ?'].join(' AND ')} ` +
                 'ORDER BY `id` LIMIT ?',
@@ -952,30 +1137,56 @@ export class DurableSubscriptionStore extends PuterStore {
         return rows.map(toRow);
     }
 
-    /** Delete a batch and leave no region delivering against any of it. */
-    async #reap(rows: readonly DurableSubscription[]): Promise<number> {
-        if (rows.length === 0) return 0;
+    /**
+     * Delete a batch and stop this region delivering against any of it. Other
+     * regions stop once the caller publishes the returned bumps; the deleted
+     * rows come back so the caller can end the schedules among them.
+     *
+     * `stillDue` re-checks candidates read from a replica on the primary, so a
+     * row resumed or extended since is kept and not reported.
+     */
+    async #reap(
+        candidates: readonly DurableSubscription[],
+        stillDue?: { where: string; values: unknown[] },
+    ): Promise<DurableReap> {
+        if (candidates.length === 0) return { rows: [], bumps: [] };
 
+        const ids = candidates.map((row) => row.subId);
+        const inList = `(${ids.map(() => '?').join(', ')})`;
         await this.clients.db.write(
-            `DELETE FROM \`${TABLE}\` WHERE \`sub_id\` IN ` +
-                `(${rows.map(() => '?').join(', ')})`,
-            rows.map((row) => row.subId),
+            `DELETE FROM \`${TABLE}\` WHERE \`sub_id\` IN ${inList}` +
+                (stillDue ? ` AND ${stillDue.where}` : ''),
+            [...ids, ...(stillDue?.values ?? [])],
         );
-
-        await this.stores.eventSubscription.dropDurables(rows);
-        for (const row of rows) {
-            // Whatever was still owed to a row that no longer exists.
-            await this.stores.pendingDelivery.purge(row.subId).catch(() => {});
+        let rows = candidates;
+        if (stillDue) {
+            const kept = new Set(
+                (
+                    await this.clients.db.pread(
+                        `SELECT \`sub_id\` FROM \`${TABLE}\` WHERE \`sub_id\` IN ${inList}`,
+                        ids,
+                    )
+                ).map((row) => String(row.sub_id)),
+            );
+            rows = candidates.filter((row) => !kept.has(row.subId));
+            if (rows.length === 0) return { rows: [], bumps: [] };
         }
-        const owners = new Set(rows.map((row) => row.ownerUserId));
-        for (const ownerUserId of owners) await this.#bump(ownerUserId);
-        return rows.length;
-    }
 
-    #assertContext(context: string | null): void {
-        if (context === null) return;
-        if (Buffer.byteLength(context, 'utf8') > DURABLE_CONTEXT_MAX_BYTES)
-            throw contextTooLarge();
+        const cached = rows.filter((row) => !isScheduleRow(row));
+        await this.stores.eventSubscription.dropDurables(cached);
+        for (const row of cached) {
+            // Whatever was still owed to a row that no longer exists.
+            try {
+                await this.stores.pendingDelivery.purge(row.subId);
+            } catch {
+                // The pending sweep purges it once it finds the row gone.
+            }
+        }
+        const owners = new Set(cached.map((row) => row.ownerUserId));
+        const bumps: GenerationBump[] = [];
+        for (const ownerUserId of owners)
+            bumps.push(await this.#bump(ownerUserId));
+        return { rows: [...rows], bumps };
     }
 
     async #bump(ownerUserId: number): Promise<GenerationBump> {

src/backend/stores/events/EventHandlerStore.mysql.test.ts

@@ -0,0 +1,392 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The user-scope handler schema and queries against MySQL 8.4. Runs only when
+ * `PUTER_TEST_MYSQL_PORT` points at a server (root / `puter`).
+ */
+
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import mysql from 'mysql2/promise';
+import { v4 as uuidv4 } from 'uuid';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import { splitMysqlStatements } from '../../clients/database/splitMysqlStatements.js';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import { appHandlerScope, hashContent } from './EventHandlerStore.js';
+
+const connection = {
+    host: '127.0.0.1',
+    port: Number(process.env.PUTER_TEST_MYSQL_PORT ?? 3307),
+    user: 'root',
+    password: 'puter',
+};
+const database = `event_user_handlers_${Date.now().toString(36)}`;
+const migrationDir = fileURLToPath(
+    new URL('../../clients/database/migrations/mysql', import.meta.url),
+);
+const SOURCE = 'async ({ event }) => { console.log(event.path); }';
+
+describe.skipIf(!process.env.PUTER_TEST_MYSQL_PORT)(
+    'user-scoped handlers (MySQL)',
+    () => {
+        let server: PuterServer;
+        let conn: mysql.Connection;
+        const userIds: number[] = [];
+
+        const handlers = () => server.stores.eventHandler;
+        const rows = async (sql: string, params: unknown[] = []) =>
+            (await conn.query(sql, params))[0] as Record<string, unknown>[];
+
+        const applyMig55 = async () => {
+            const sql = readFileSync(
+                `${migrationDir}/mysql_mig_55.sql`,
+                'utf8',
+            );
+            for (const statement of splitMysqlStatements(sql))
+                await conn.query(statement);
+        };
+
+        const bind = async (
+            holder: number,
+            app: string,
+            name: string,
+            handlerScope: number,
+            suspended = false,
+        ) => {
+            await conn.query(
+                'INSERT INTO `event_subscriptions` (`sub_id`, `token`, `owner_user_id`, `holder_user_id`, `app_uid`, `subject`, `anchor_uid`, `anchor_path`, `delivery`, `handler_name`, `handler_scope`, `targets`, `permission`, `suspended_at`, `suspended_reason`, `created_at`) ' +
+                    "VALUES (?, ?, ?, ?, ?, 'fs:/x', 'a', '/x', 'single', ?, ?, '[]', 'list', ?, ?, 1)",
+                [
+                    uuidv4(),
+                    `f#${uuidv4()}`,
+                    holder,
+                    holder,
+                    app,
+                    name,
+                    handlerScope,
+                    suspended ? 1 : null,
+                    suspended ? 'handler_not_found' : null,
+                ],
+            );
+        };
+
+        /**
+         * The `key` column of every table in the plan of the last statement a
+         * spy saw.
+         */
+        const planKeys = async (spy: {
+            mock: { calls: unknown[][] };
+        }): Promise<Record<string, string | null>> => {
+            const [statement, params] = spy.mock.calls.at(-1) as [
+                string,
+                unknown[],
+            ];
+            const plan = await rows(`EXPLAIN ${statement}`, params);
+            return Object.fromEntries(
+                plan.map((row) => [
+                    String(row.table),
+                    (row.key as string | null) ?? null,
+                ]),
+            );
+        };
+
+        beforeAll(async () => {
+            const admin = await mysql.createConnection(connection);
+            await admin.query(`CREATE DATABASE \`${database}\``);
+            await admin.end();
+            server = await setupTestServer({
+                database: {
+                    engine: 'mysql',
+                    ...connection,
+                    database,
+                    migrationPaths: [migrationDir],
+                },
+                events: { enabled: true },
+            } as never);
+            conn = await mysql.createConnection({ ...connection, database });
+            for (let i = 0; i < 12; i++) {
+                const user = await server.stores.user.create({
+                    username: `ush-${uuidv4().slice(0, 8)}`,
+                    uuid: uuidv4(),
+                    password: null,
+                    email: null,
+                });
+                userIds.push(user.id);
+            }
+        }, 240_000);
+
+        afterAll(async () => {
+            await conn?.end();
+            await server?.shutdown();
+            const admin = await mysql.createConnection(connection);
+            await admin.query(`DROP DATABASE IF EXISTS \`${database}\``);
+            await admin.end();
+        });
+
+        it('runs on MySQL 8.4', async () => {
+            const [row] = await rows('SELECT VERSION() AS version');
+            expect(String(row!.version)).toMatch(/^8\.4\./);
+        });
+
+        it('applies over existing subscription rows, and a replay is a no-op', async () => {
+            // Back to the state before the migration, with rows already in.
+            await conn.query('DROP TABLE `event_user_handlers`');
+            await conn.query(
+                'ALTER TABLE `event_subscriptions` DROP COLUMN `handler_scope`',
+            );
+            await conn.query('DELETE FROM `event_subscriptions`');
+            for (const name of ['a', 'b', 'c'])
+                await conn.query(
+                    'INSERT INTO `event_subscriptions` (`sub_id`, `token`, `owner_user_id`, `holder_user_id`, `app_uid`, `subject`, `anchor_uid`, `anchor_path`, `delivery`, `handler_name`, `targets`, `permission`, `created_at`) ' +
+                        "VALUES (?, ?, ?, ?, 'app-old', 'fs:/x', 'a', '/x', 'single', ?, '[]', 'list', 1)",
+                    [uuidv4(), `f#${uuidv4()}`, userIds[0], userIds[0], name],
+                );
+
+            await applyMig55();
+            const scopes = async () =>
+                (
+                    await rows(
+                        'SELECT `handler_scope` FROM `event_subscriptions`',
+                    )
+                ).map((row) => Number(row.handler_scope));
+            expect(await scopes()).toEqual([0, 0, 0]);
+
+            await applyMig55();
+            expect(await scopes()).toEqual([0, 0, 0]);
+
+            const [column] = await rows(
+                "SELECT COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'event_subscriptions' AND COLUMN_NAME = 'handler_scope'",
+            );
+            expect(column).toMatchObject({
+                COLUMN_TYPE: 'tinyint unsigned',
+                IS_NULLABLE: 'NO',
+                COLUMN_DEFAULT: '0',
+            });
+
+            const indexes = await rows(
+                "SELECT INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'event_user_handlers' ORDER BY INDEX_NAME, SEQ_IN_INDEX",
+            );
+            const byIndex = (name: string) =>
+                indexes
+                    .filter((row) => row.INDEX_NAME === name)
+                    .map((row) => String(row.COLUMN_NAME));
+            expect(byIndex('idx_event_user_handlers_app_holder_name')).toEqual([
+                'app_uid',
+                'holder_user_id',
+                'name',
+            ]);
+            expect(byIndex('idx_event_user_handlers_holder_app')).toEqual([
+                'holder_user_id',
+                'app_uid',
+            ]);
+            expect(
+                indexes.find(
+                    (row) =>
+                        row.INDEX_NAME ===
+                        'idx_event_user_handlers_app_holder_name',
+                ),
+            ).toMatchObject({ NON_UNIQUE: 0 });
+
+            const [appUidColumn] = await rows(
+                "SELECT CHARACTER_SET_NAME, COLLATION_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'event_user_handlers' AND COLUMN_NAME = 'app_uid'",
+            );
+            expect(appUidColumn).toMatchObject({
+                CHARACTER_SET_NAME: 'latin1',
+                COLLATION_NAME: 'latin1_swedish_ci',
+            });
+        });
+
+        it('keeps the same name once in the app scope and once per holder', async () => {
+            const app = `app-${uuidv4()}`;
+            const insert = (holder: number) =>
+                conn.query(
+                    'INSERT INTO `event_user_handlers` (`app_uid`, `holder_user_id`, `name`, `source`, `source_hash`, `created_at`, `updated_at`, `scope_since`) VALUES (?, ?, ?, ?, ?, 1, 1, 1)',
+                    [app, holder, 'daily', SOURCE, hashContent(SOURCE)],
+                );
+            await handlers().publish({
+                scope: appHandlerScope(app),
+                name: 'daily',
+                source: SOURCE,
+            });
+            await insert(userIds[1]!);
+            await insert(userIds[2]!);
+            await expect(insert(userIds[1]!)).rejects.toMatchObject({
+                code: 'ER_DUP_ENTRY',
+            });
+        });
+
+        it('runs the store against both tables', async () => {
+            const app = `app-${uuidv4()}`;
+            const [a, b] = [userIds[3]!, userIds[4]!];
+            const scope = (holder: number) => ({ appUid: app, holder });
+            await handlers().publish({
+                scope: appHandlerScope(app),
+                name: 'daily',
+                source: SOURCE,
+            });
+            for (const holder of [a, b])
+                await handlers().publish({
+                    scope: scope(holder),
+                    name: 'daily',
+                    source: `${SOURCE} // ${holder}`,
+                });
+            await handlers().publish({
+                scope: scope(a),
+                name: `@${hashContent(SOURCE)}`,
+                source: SOURCE,
+            });
+
+            expect(await handlers().countForApp(scope(a))).toBe(2);
+            expect(await handlers().countForApp(appHandlerScope(app))).toBe(1);
+            expect(await handlers().totalSourceBytesForApp(scope(b))).toBe(
+                Buffer.byteLength(`${SOURCE} // ${b}`),
+            );
+            expect(
+                (await handlers().setForApp(scope(a), { replica: true })).map(
+                    (h) => h.name,
+                ),
+            ).toEqual([`@${hashContent(SOURCE)}`, 'daily']);
+            expect(await handlers().countUserScopesWithHandlersForUser(a)).toBe(
+                1,
+            );
+            expect(
+                await handlers().countUserScopesWithHandlersForUser(a, {
+                    createdBefore: 1_000,
+                }),
+            ).toBe(0);
+            expect(
+                await handlers().countUserScopesWithHandlersForUser(a, {
+                    createdBefore: Date.now() + 60_000,
+                }),
+            ).toBe(1);
+            const page = await handlers().listUserScopesForUser(a);
+            expect(page.total).toBe(1);
+            expect(page.items).toEqual([
+                expect.objectContaining({ appUid: app, handlerCount: 2 }),
+            ]);
+            expect(
+                (await handlers().listUserScopes(app)).items.map(
+                    (s) => s.holder,
+                ),
+            ).toEqual([a, b].sort((x, y) => x - y));
+
+            await bind(a, app, 'daily', 1);
+            await bind(b, app, 'daily', 1, true);
+            await bind(a, app, 'daily', 0);
+            expect(
+                await handlers().countSubscriptionsByHandler(scope(a)),
+            ).toEqual(new Map([['daily', 1]]));
+            expect(
+                await handlers().countSubscriptionsByHandler(
+                    appHandlerScope(app),
+                ),
+            ).toEqual(new Map([['daily', 1]]));
+
+            await conn.query(
+                'UPDATE `event_user_handlers` SET `updated_at` = 1 WHERE `app_uid` = ?',
+                [app],
+            );
+            const found = await handlers().listUnboundUserHandlers(1_000);
+            expect(
+                found.items
+                    .filter((item) => item.appUid === app)
+                    .map((item) => [item.holder, item.name]),
+            ).toEqual([[a, `@${hashContent(SOURCE)}`]]);
+            await expect(
+                handlers().deleteUserHandlerIfUnbound(scope(a), 'daily', 1_000),
+            ).resolves.toBe(false);
+            await expect(
+                handlers().deleteUserHandlerIfUnbound(
+                    scope(a),
+                    `@${hashContent(SOURCE)}`,
+                    1_000,
+                ),
+            ).resolves.toBe(true);
+
+            await expect(handlers().deleteUserScopesForHolder(b)).resolves.toBe(
+                1,
+            );
+            await expect(handlers().deleteForApp(app)).resolves.toBe(2);
+            expect(await handlers().countForApp(scope(a))).toBe(0);
+        });
+
+        it('serves the user-scope reads from the holder-leading indexes', async () => {
+            const app = `app-${uuidv4()}`;
+            // A popular handler: many holders on one (app, name), so the
+            // (app_uid, handler_name) index is the wrong one to pick.
+            for (const holder of userIds)
+                for (let i = 0; i < 80; i++)
+                    await bind(holder, app, 'popular', 1);
+            await handlers().publish({
+                scope: { appUid: app, holder: userIds[0]! },
+                name: 'popular',
+                source: SOURCE,
+            });
+            await conn.query('ANALYZE TABLE `event_subscriptions`');
+            await conn.query('ANALYZE TABLE `event_user_handlers`');
+
+            const pread = vi.spyOn(server.clients.db, 'pread');
+            const read = vi.spyOn(server.clients.db, 'read');
+            try {
+                await server.stores.durableSubscription.listByHandler(
+                    app,
+                    'user',
+                    userIds[0]!,
+                    'popular',
+                );
+                expect(await planKeys(pread)).toEqual({
+                    event_subscriptions: 'idx_event_subscriptions_holder',
+                });
+
+                await handlers().countSubscriptionsByHandler({
+                    appUid: app,
+                    holder: userIds[0]!,
+                });
+                expect(await planKeys(read)).toEqual({
+                    event_subscriptions: 'idx_event_subscriptions_holder',
+                });
+
+                await handlers().countUserScopesWithHandlersForUser(
+                    userIds[0]!,
+                    {
+                        createdBefore: Date.now(),
+                    },
+                );
+                expect((await planKeys(pread)).event_user_handlers).toBe(
+                    'idx_event_user_handlers_holder_app',
+                );
+
+                await handlers().listUserScopesForUser(userIds[0]!);
+                expect((await planKeys(read)).event_user_handlers).toBe(
+                    'idx_event_user_handlers_holder_app',
+                );
+
+                await handlers().listUnboundUserHandlers(1_000);
+                expect((await planKeys(read)).s).toBe(
+                    'idx_event_subscriptions_holder',
+                );
+            } finally {
+                pread.mockRestore();
+                read.mockRestore();
+            }
+        }, 120_000);
+    },
+);

src/backend/stores/events/EventHandlerStore.ts

@@ -21,6 +21,7 @@ import { createHash } from 'node:crypto';
 import {
     EVENTS_HANDLERS_PER_APP,
     EVENTS_HANDLER_SOURCE_MAX_BYTES,
+    EVENTS_USER_HANDLERS_PER_SCOPE,
 } from '../../controllers/events/limits.js';
 import { HttpError } from '../../core/http/HttpError.js';
 import {
@@ -47,6 +48,7 @@ import { PuterStore } from '../types.js';
  */
 
 const TABLE = 'event_handlers';
+const USER_TABLE = 'event_user_handlers';
 const SUBSCRIPTION_TABLE = 'event_subscriptions';
 const APP_TABLE = 'apps';
 
@@ -57,12 +59,24 @@ const APP_TABLE = 'apps';
 export const EVENTS_WORKERS_LIST_DEFAULT_LIMIT = 50;
 export const EVENTS_WORKERS_LIST_LIMIT_CAP = 200;
 
+/** Default page of `listUserScopes`, and of one GC read. */
+const USER_SCOPES_PAGE_DEFAULT = 500;
+const USER_SCOPES_PAGE_CAP = 1000;
+const UNBOUND_HANDLERS_BATCH = 200;
+
 /** Longest a handler name may be, matching the column that holds it. */
 export const HANDLER_NAME_MAX_LENGTH = 128;
 
 /** Names are addressable identifiers, not free text. */
 const NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/;
 
+/**
+ * A server-minted name for an inline handler: `@` and the source hash. `@` is
+ * outside {@link NAME_PATTERN}, so a minted name never collides with a chosen
+ * one, and only a user scope accepts it.
+ */
+const INLINE_NAME_PATTERN = /^@[0-9a-f]{64}$/;
+
 /**
  * Longest a delivery may address an app's handler set from cache. Every write
  * here drops the entry on every node, so this only bounds a read that raced a
@@ -70,8 +84,66 @@ const NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/;
  */
 export const HANDLER_SET_CACHE_TTL_SECONDS = 30;
 
-const handlerSetCacheKey = (appUid: string): string =>
-    `eventHandlers:set:${appUid}`;
+// -- Scopes -----------------------------------------------------------
+
+/**
+ * Whose handlers a call is about: an app's own set (`holder` 0), or the
+ * handlers one user published into their scope of that app. The two live in
+ * separate tables, so code that predates user scopes never reads one.
+ */
+export interface HandlerScope {
+    appUid: string;
+    /** `users.id` of the scope's holder, or 0 for the app's own set. */
+    holder: number;
+}
+
+/** The app's own scope. */
+export const appHandlerScope = (appUid: string): HandlerScope => ({
+    appUid,
+    holder: 0,
+});
+
+const isUserScope = (scope: HandlerScope): boolean => scope.holder !== 0;
+
+const assertScope = (scope: HandlerScope): void => {
+    if (!Number.isInteger(scope.holder) || scope.holder < 0)
+        throw new TypeError('handler scope holder must be a user id or 0');
+};
+
+const tableOf = (scope: HandlerScope): string =>
+    isUserScope(scope) ? USER_TABLE : TABLE;
+
+const handlerCapOf = (scope: HandlerScope): number =>
+    isUserScope(scope)
+        ? EVENTS_USER_HANDLERS_PER_SCOPE
+        : EVENTS_HANDLERS_PER_APP;
+
+/** The WHERE clause for a scope's rows and its parameters. */
+const scopeFilter = (
+    scope: HandlerScope,
+): { where: string; params: unknown[] } => {
+    assertScope(scope);
+    return isUserScope(scope)
+        ? {
+              where: '`app_uid` = ? AND `holder_user_id` = ?',
+              params: [scope.appUid, scope.holder],
+          }
+        : { where: '`app_uid` = ?', params: [scope.appUid] };
+};
+
+// App scope keeps the key it always had, so nodes on either side of a rolling
+// deploy share it.
+const handlerSetCacheKey = (scope: HandlerScope): string =>
+    isUserScope(scope)
+        ? `eventHandlers:uset:${scope.appUid}:${scope.holder}`
+        : `eventHandlers:set:${scope.appUid}`;
+
+/**
+ * The same set as read from the replica, kept apart so a lagging read can never
+ * stand in for the primary's answer on the delivery path.
+ */
+const replicaSetCacheKey = (scope: HandlerScope): string =>
+    `${handlerSetCacheKey(scope)}:r`;
 
 // -- Shapes -----------------------------------------------------------
 
@@ -119,8 +191,45 @@ export interface ListEventsWorkersOptions {
     appUid?: string;
 }
 
-export interface PublishHandlerInput {
+/** One user scope: the handlers a user keeps in one app. */
+export interface UserScopeSummary {
     appUid: string;
+    /** Empty when the app row is gone. */
+    appName: string;
+    appTitle: string;
+    handlerCount: number;
+    /** Earliest handler `created_at` in the scope. */
+    createdAt: number;
+    /** Latest handler `updated_at` in the scope. */
+    updatedAt: number;
+}
+
+/** A user scope as an app-wide listing reports it. */
+export interface AppUserScope {
+    holder: number;
+    handlerCount: number;
+    createdAt: number;
+    updatedAt: number;
+}
+
+/** One user-scope handler a GC pass may delete. */
+export interface UnboundUserHandler {
+    id: number;
+    appUid: string;
+    holder: number;
+    name: string;
+}
+
+/**
+ * One entry of a scope's handler set. `updatedAt` (unix seconds) is set for a
+ * user scope and absent from entries cached without it, meaning unknown.
+ */
+export type HandlerSetEntry = Pick<EventHandler, 'name' | 'sourceHash'> & {
+    updatedAt?: number;
+};
+
+export interface PublishHandlerInput {
+    scope: HandlerScope;
     name: string;
     source: string;
     /**
@@ -130,6 +239,11 @@ export interface PublishHandlerInput {
     ifHash?: string | null;
     /** Take the name whatever is published under it. */
     replace?: boolean;
+    /**
+     * A create leaves peer nodes' cached sets alone: one missing the new name
+     * is re-read when asked for it. Updates always broadcast.
+     */
+    broadcast?: boolean;
 }
 
 /** What one publish did, which is what the caller reports back. */
@@ -162,10 +276,12 @@ const invalidSource = (): HttpError =>
         legacyCode: 'events_handler_source_invalid',
     });
 
-const tooManyHandlers = (): HttpError =>
+const tooManyHandlers = (scope: HandlerScope): HttpError =>
     new HttpError(
         429,
-        `An app may publish ${EVENTS_HANDLERS_PER_APP} handlers`,
+        isUserScope(scope)
+            ? `A user scope may hold ${EVENTS_USER_HANDLERS_PER_SCOPE} handlers`
+            : `An app may publish ${EVENTS_HANDLERS_PER_APP} handlers`,
         { legacyCode: 'events_handler_limit' },
     );
 
@@ -204,10 +320,11 @@ const SELECT_COLUMNS =
 export const hashContent = (content: string): string =>
     createHash('sha256').update(content, 'utf8').digest('hex');
 
-export const isValidHandlerName = (name: string): boolean =>
+/** `userScope` also admits the `@<sha256>` names minted for inline handlers. */
+export const isValidHandlerName = (name: string, userScope = false): boolean =>
     name.length > 0 &&
     name.length <= HANDLER_NAME_MAX_LENGTH &&
-    NAME_PATTERN.test(name);
+    (NAME_PATTERN.test(name) || (userScope && INLINE_NAME_PATTERN.test(name)));
 
 export class EventHandlerStore extends PuterStore {
     // -- Writes ------------------------------------------------------
@@ -220,11 +337,12 @@ export class EventHandlerStore extends PuterStore {
      * with a row that no longer exists.
      */
     async publish(input: PublishHandlerInput): Promise<PublishResult> {
-        const name = this.#assertName(input.name);
+        const { scope } = input;
+        const name = this.#assertName(input.name, scope);
         const source = this.#assertSource(input.source);
         const sourceHash = hashContent(source);
 
-        const existing = await this.getByName(input.appUid, name);
+        const existing = await this.getByName(scope, name);
         if (!existing) {
             // `ifHash` naming a source that is not there is the same race from
             // the other side: something removed the row this publish was
@@ -232,10 +350,11 @@ export class EventHandlerStore extends PuterStore {
             if (input.ifHash && !input.replace) throw publishConflict(name);
             return {
                 handler: await this.#insert(
-                    input.appUid,
+                    scope,
                     name,
                     source,
                     sourceHash,
+                    input.broadcast !== false,
                 ),
                 outcome: 'created',
             };
@@ -253,34 +372,155 @@ export class EventHandlerStore extends PuterStore {
         if (!input.replace && !basedOnPublished) throw publishConflict(name);
 
         return {
-            handler: await this.#update(existing, source, sourceHash),
+            handler: await this.#update(scope, existing, source, sourceHash),
             outcome: 'updated',
         };
     }
 
-    /** Drop one handler. Null when the app had nothing published by that name. */
-    async remove(appUid: string, name: string): Promise<EventHandler | null> {
-        const existing = await this.getByName(appUid, name);
+    /** Drop one handler. Null when the scope had nothing published by that name. */
+    async remove(
+        scope: HandlerScope,
+        name: string,
+    ): Promise<EventHandler | null> {
+        const existing = await this.getByName(scope, name);
         if (!existing) return null;
+        const { where, params } = scopeFilter(scope);
         await this.clients.db.write(
-            `DELETE FROM \`${TABLE}\` WHERE \`app_uid\` = ? AND \`name\` = ?`,
-            [appUid, name],
+            `DELETE FROM \`${tableOf(scope)}\` WHERE ${where} AND \`name\` = ?`,
+            [...params, name],
         );
-        await this.#forgetSet(appUid);
+        await this.#forgetSet(scope);
         return existing;
     }
 
     /**
-     * Drop every handler an app has published — what a deleted app leaves
-     * behind.
+     * Drop every handler an app has published, in its own set and in every
+     * user's scope — what a deleted app leaves behind. Returns the rows
+     * removed. User scopes go a batch of holders at a time, each batch's cached
+     * sets dropped here and on every peer.
      */
     async deleteForApp(appUid: string): Promise<number> {
         const result = await this.clients.db.write(
             `DELETE FROM \`${TABLE}\` WHERE \`app_uid\` = ?`,
             [appUid],
         );
-        await this.#forgetSet(appUid);
-        return result.affectedRows;
+        let removed = result.affectedRows ?? 0;
+        for (;;) {
+            const holders = (
+                await this.clients.db.pread(
+                    `SELECT DISTINCT \`holder_user_id\` FROM \`${USER_TABLE}\` ` +
+                        'WHERE `app_uid` = ? ORDER BY `holder_user_id` LIMIT ?',
+                    [appUid, USER_SCOPES_PAGE_DEFAULT],
+                )
+            ).map((row) => Number(row.holder_user_id));
+            if (holders.length === 0) break;
+            const batch = await this.clients.db.write(
+                `DELETE FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? ` +
+                    `AND \`holder_user_id\` IN (${holders.map(() => '?').join(', ')})`,
+                [appUid, ...holders],
+            );
+            await this.#forgetSets(
+                holders.map((holder) => ({ appUid, holder })),
+            );
+            const affected = batch.affectedRows ?? 0;
+            removed += affected;
+            if (affected === 0) break;
+        }
+        await this.#forgetSet(appHandlerScope(appUid));
+        return removed;
+    }
+
+    /** Drop one user's scope of one app. Returns the handlers removed. */
+    async deleteUserScope(appUid: string, holder: number): Promise<number> {
+        const scope = { appUid, holder };
+        if (!isUserScope(scope)) return 0;
+        assertScope(scope);
+        const result = await this.clients.db.write(
+            `DELETE FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? AND \`holder_user_id\` = ?`,
+            [appUid, holder],
+        );
+        await this.#forgetSet(scope);
+        return result.affectedRows ?? 0;
+    }
+
+    /**
+     * Drop every scope a user holds, across apps — what an account deletion
+     * leaves behind. Returns the handlers removed.
+     */
+    async deleteUserScopesForHolder(userId: number): Promise<number> {
+        if (!Number.isInteger(userId) || userId <= 0) return 0;
+        const apps = await this.clients.db.pread(
+            `SELECT DISTINCT \`app_uid\` FROM \`${USER_TABLE}\` WHERE \`holder_user_id\` = ?`,
+            [userId],
+        );
+        const result = await this.clients.db.write(
+            `DELETE FROM \`${USER_TABLE}\` WHERE \`holder_user_id\` = ?`,
+            [userId],
+        );
+        for (const row of apps)
+            await this.#forgetSet({
+                appUid: String(row.app_uid),
+                holder: userId,
+            });
+        return result.affectedRows ?? 0;
+    }
+
+    /**
+     * Delete one user-scope handler if no subscription row of any state is
+     * bound to it and it was last touched before `olderThan` (unix seconds).
+     * The check and the delete are one statement, so a row bound in between
+     * keeps the handler. True when it deleted.
+     */
+    async deleteUserHandlerIfUnbound(
+        scope: HandlerScope,
+        name: string,
+        olderThan: number,
+    ): Promise<boolean> {
+        if (!isUserScope(scope)) return false;
+        assertScope(scope);
+        const result = await this.clients.db.write(
+            `DELETE FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? AND \`holder_user_id\` = ? ` +
+                'AND `name` = ? AND `updated_at` < ? ' +
+                `AND NOT EXISTS (SELECT 1 FROM \`${SUBSCRIPTION_TABLE}\` ` +
+                'WHERE `holder_user_id` = ? AND `app_uid` = ? ' +
+                'AND `handler_scope` = 1 AND `handler_name` = ?)',
+            [
+                scope.appUid,
+                scope.holder,
+                name,
+                olderThan,
+                scope.holder,
+                scope.appUid,
+                name,
+            ],
+        );
+        if (!result.anyRowsAffected) return false;
+        await this.#forgetSet(scope);
+        return true;
+    }
+
+    /**
+     * Mark one user-scope handler as just used, so the unbound sweep (which
+     * only takes rows last touched before its cutoff) leaves it alone. False
+     * when the handler is gone.
+     */
+    async touchUserHandler(
+        scope: HandlerScope,
+        name: string,
+    ): Promise<boolean> {
+        if (!isUserScope(scope)) return false;
+        assertScope(scope);
+        const result = await this.clients.db.write(
+            `UPDATE \`${USER_TABLE}\` SET \`updated_at\` = ? ` +
+                'WHERE `app_uid` = ? AND `holder_user_id` = ? AND `name` = ?',
+            [nowSeconds(), scope.appUid, scope.holder, name],
+        );
+        return Boolean(result.anyRowsAffected);
+    }
+
+    /** Drop this region's cached sets of a scope, without telling peers. */
+    async forgetScopeCache(scope: HandlerScope): Promise<void> {
+        await this.#forgetSet(scope, false);
     }
 
     // -- Reads -------------------------------------------------------
@@ -292,34 +532,36 @@ export class EventHandlerStore extends PuterStore {
      * absent.
      */
     async getByName(
-        appUid: string,
+        scope: HandlerScope,
         name: string,
     ): Promise<EventHandler | null> {
+        const { where, params } = scopeFilter(scope);
         const rows = await this.clients.db.pread(
-            `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
-                'WHERE `app_uid` = ? AND `name` = ?',
-            [appUid, name],
+            `SELECT ${SELECT_COLUMNS} FROM \`${tableOf(scope)}\` ` +
+                `WHERE ${where} AND \`name\` = ?`,
+            [...params, name],
         );
         return rows.length > 0 ? toRow(rows[0]) : null;
     }
 
     /**
-     * What an app has published, with how many subscriptions each name is
-     * carrying. Bounded by the per-app cap, so the whole set is one page.
+     * What a scope has published, with how many subscriptions each name is
+     * carrying. Bounded by the scope's cap, so the whole set is one page.
      *
      * The counts come from one grouped read rather than a join, because the
-     * subscription side has to count names an app never published — a handler
+     * subscription side has to count names a scope never published — a handler
      * removed while subscriptions still pointed at it leaves exactly that.
      */
-    async listForApp(appUid: string): Promise<EventHandlerSummary[]> {
+    async listForApp(scope: HandlerScope): Promise<EventHandlerSummary[]> {
+        const { where, params } = scopeFilter(scope);
         const rows = await this.clients.db.read(
             'SELECT `name`, `source_hash`, `updated_at` ' +
-                `FROM \`${TABLE}\` WHERE \`app_uid\` = ? ` +
+                `FROM \`${tableOf(scope)}\` WHERE ${where} ` +
                 'ORDER BY `name` LIMIT ?',
-            [appUid, EVENTS_HANDLERS_PER_APP],
+            [...params, handlerCapOf(scope)],
         );
 
-        const counts = await this.countSubscriptionsByHandler(appUid);
+        const counts = await this.countSubscriptionsByHandler(scope);
         return rows.map((row) => ({
             name: String(row.name),
             hash: String(row.source_hash),
@@ -329,24 +571,30 @@ export class EventHandlerStore extends PuterStore {
     }
 
     /**
-     * Name and source hash for every handler an app has published: the identity
-     * of its set, without reading a byte of source. What a deploy checks the
-     * script it was asked for against, and what the delivery path's cache is
-     * filled from. Primary, so neither reads a set a replica has not caught up
-     * to.
+     * Name and source hash for every handler a scope has published: the
+     * identity of its set, without reading a byte of source. What a deploy
+     * checks the script it was asked for against, and what the delivery path's
+     * cache is filled from. Primary, so neither reads a set a replica has not
+     * caught up to.
      */
     async setForApp(
-        appUid: string,
-    ): Promise<Pick<EventHandler, 'name' | 'sourceHash'>[]> {
-        const rows = await this.clients.db.pread(
-            'SELECT `name`, `source_hash` ' +
-                `FROM \`${TABLE}\` WHERE \`app_uid\` = ? ` +
-                'ORDER BY `name` LIMIT ?',
-            [appUid, EVENTS_HANDLERS_PER_APP],
-        );
+        scope: HandlerScope,
+        options: { replica?: boolean } = {},
+    ): Promise<HandlerSetEntry[]> {
+        const { where, params: whereParams } = scopeFilter(scope);
+        const withAge = isUserScope(scope);
+        const sql =
+            `SELECT \`name\`, \`source_hash\`${withAge ? ', `updated_at`' : ''} ` +
+            `FROM \`${tableOf(scope)}\` WHERE ${where} ` +
+            'ORDER BY `name` LIMIT ?';
+        const params = [...whereParams, handlerCapOf(scope)];
+        const rows = options.replica
+            ? await this.clients.db.read(sql, params)
+            : await this.clients.db.pread(sql, params);
         return rows.map((row) => ({
             name: String(row.name),
             sourceHash: String(row.source_hash),
+            ...(withAge ? { updatedAt: Number(row.updated_at) || 0 } : {}),
         }));
     }
 
@@ -354,23 +602,43 @@ export class EventHandlerStore extends PuterStore {
      * `setForApp` for the delivery path, which asks once per invocation. Cached
      * for up to {@link HANDLER_SET_CACHE_TTL_SECONDS}, and dropped everywhere by
      * every write here. An empty set is not cached: it fails the delivery, and
-     * must not outlast the publish that fills it.
+     * must not outlast the publish that fills it. Schedule runs read a miss
+     * from the replica (`missRead: 'replica'`), since they come by the clock
+     * rather than after a write, and cache it under a key of their own; an
+     * empty replica read is checked on the primary, since a lagging replica
+     * right after a publish would fail the run. Called by extensions.
      */
     async cachedSetForApp(
-        appUid: string,
-    ): Promise<Pick<EventHandler, 'name' | 'sourceHash'>[]> {
-        const key = handlerSetCacheKey(appUid);
+        scope: HandlerScope,
+        options: {
+            missRead?: 'primary' | 'replica';
+            handlerName?: string;
+        } = {},
+    ): Promise<HandlerSetEntry[]> {
+        const replica = options.missRead === 'replica';
+        // A set without the handler may be from before its publish.
+        const usable = (set: Pick<EventHandler, 'name'>[]) =>
+            set.length > 0 &&
+            (options.handlerName === undefined ||
+                set.some((handler) => handler.name === options.handlerName));
+        const key = replica
+            ? replicaSetCacheKey(scope)
+            : handlerSetCacheKey(scope);
         try {
             const cached = JSON.parse(
                 (await this.clients.redis.get(key)) ?? 'null',
             ) as unknown;
-            if (Array.isArray(cached) && cached.length > 0)
-                return cached as Pick<EventHandler, 'name' | 'sourceHash'>[];
+            if (
+                Array.isArray(cached) &&
+                usable(cached as Pick<EventHandler, 'name'>[])
+            )
+                return cached as HandlerSetEntry[];
         } catch {
             // Read the table instead.
         }
 
-        const set = await this.setForApp(appUid);
+        let set = await this.setForApp(scope, { replica });
+        if (replica && !usable(set)) set = await this.setForApp(scope);
         if (set.length > 0) {
             try {
                 await this.clients.redis.set(
@@ -387,57 +655,70 @@ export class EventHandlerStore extends PuterStore {
     }
 
     /**
-     * Every handler the app has published, sources included — what the events
+     * Every handler the scope has published, sources included — what the events
      * worker is generated from. Primary, for the same reason `getByName` is:
      * the bake follows the publish that triggered it.
      */
-    async allForApp(appUid: string): Promise<EventHandler[]> {
+    async allForApp(scope: HandlerScope): Promise<EventHandler[]> {
+        const { where, params } = scopeFilter(scope);
         const rows = await this.clients.db.pread(
-            `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
-                'WHERE `app_uid` = ? ORDER BY `name` LIMIT ?',
-            [appUid, EVENTS_HANDLERS_PER_APP],
+            `SELECT ${SELECT_COLUMNS} FROM \`${tableOf(scope)}\` ` +
+                `WHERE ${where} ORDER BY \`name\` LIMIT ?`,
+            [...params, handlerCapOf(scope)],
         );
         return rows.map(toRow);
     }
 
-    async countForApp(appUid: string): Promise<number> {
+    async countForApp(scope: HandlerScope): Promise<number> {
+        const { where, params } = scopeFilter(scope);
         const [row] = await this.clients.db.pread(
-            `SELECT COUNT(*) AS \`total\` FROM \`${TABLE}\` WHERE \`app_uid\` = ?`,
-            [appUid],
+            `SELECT COUNT(*) AS \`total\` FROM \`${tableOf(scope)}\` WHERE ${where}`,
+            params,
         );
         return Number(row?.total ?? 0);
     }
 
     /**
-     * Bytes of source an app has published, across every handler. What a
-     * publish is checked against before it lands, so the app's generated events
+     * Bytes of source a scope has published, across every handler. What a
+     * publish is checked against before it lands, so the generated events
      * worker never grows past what the runtime may load.
      *
      * `OCTET_LENGTH`, not `LENGTH`: only MySQL measures the latter in bytes,
      * and the caller compares this against `Buffer.byteLength`.
      */
-    async totalSourceBytesForApp(appUid: string): Promise<number> {
+    async totalSourceBytesForApp(scope: HandlerScope): Promise<number> {
+        const { where, params } = scopeFilter(scope);
         const [row] = await this.clients.db.pread(
-            `SELECT SUM(OCTET_LENGTH(\`source\`)) AS \`total\` FROM \`${TABLE}\` WHERE \`app_uid\` = ?`,
-            [appUid],
+            `SELECT SUM(OCTET_LENGTH(\`source\`)) AS \`total\` FROM \`${tableOf(scope)}\` WHERE ${where}`,
+            params,
         );
         return Number(row?.total ?? 0);
     }
 
     /**
-     * How many subscriptions each of an app's handler names is carrying,
+     * How many subscriptions each of a scope's handler names is carrying,
      * suspended rows included — a suspended subscription is still a dependent,
      * and it is the reason a removal is not a delete.
+     *
+     * A user scope reads by holder, so it never scans the other holders of a
+     * popular app.
      */
     async countSubscriptionsByHandler(
-        appUid: string,
+        scope: HandlerScope,
     ): Promise<Map<string, number>> {
+        assertScope(scope);
+        const filter = isUserScope(scope)
+            ? '`holder_user_id` = ? AND `app_uid` = ? AND `handler_scope` = 1'
+            : '`app_uid` = ? AND `handler_scope` = 0';
+        const params = isUserScope(scope)
+            ? [scope.holder, scope.appUid]
+            : [scope.appUid];
         const rows = await this.clients.db.read(
             'SELECT `handler_name`, COUNT(*) AS `total` ' +
                 `FROM \`${SUBSCRIPTION_TABLE}\` ` +
-                'WHERE `app_uid` = ? AND `handler_name` IS NOT NULL ' +
+                `WHERE ${filter} AND \`handler_name\` IS NOT NULL ` +
                 'GROUP BY `handler_name`',
-            [appUid],
+            params,
         );
         return new Map(
             rows.map((row) => [
@@ -450,7 +731,8 @@ export class EventHandlerStore extends PuterStore {
     /**
      * How many of an owner's apps have at least one published handler — what a
      * rent listener counts to tell a first publish (0→1) from a redeploy, and a
-     * last removal (1→0) from one of several.
+     * last removal (1→0) from one of several. Reads the apps' own sets only:
+     * user scopes are never the owner's.
      *
      * `createdBefore` (epoch ms) restricts to apps whose events worker existed
      * before that time, by its earliest handler; used to bill only what was
@@ -482,10 +764,36 @@ export class EventHandlerStore extends PuterStore {
         return Number(rows[0]?.total ?? 0);
     }
 
+    /**
+     * How many apps a user holds a scope in — the rent count for user scopes,
+     * with `createdBefore` (epoch ms) meaning what it does for
+     * {@link countAppsWithHandlersForOwner}.
+     */
+    async countUserScopesWithHandlersForUser(
+        userId: number,
+        opts: { createdBefore?: number } = {},
+    ): Promise<number> {
+        const params: unknown[] = [userId];
+        let having = '';
+        if (opts.createdBefore !== undefined) {
+            having = ' HAVING MIN(`scope_since`) < ?';
+            params.push(Math.ceil(opts.createdBefore / 1000));
+        }
+        const rows = await this.clients.db.pread(
+            'SELECT COUNT(*) AS `total` FROM (' +
+                `SELECT \`app_uid\` FROM \`${USER_TABLE}\` ` +
+                `WHERE \`holder_user_id\` = ? GROUP BY \`app_uid\`${having}` +
+                ') AS `counted`',
+            params,
+        );
+        return Number(rows[0]?.total ?? 0);
+    }
+
     /**
      * The events workers an owner is running, one row per app with at least one
      * published handler. Keyset-paginated on `app_uid`, since the grouping key
-     * is the only stable order a set this shape can offer.
+     * is the only stable order a set this shape can offer. Reads the apps' own
+     * sets only.
      */
     async listEventsWorkersForOwner(
         ownerUserId: number,
@@ -544,21 +852,184 @@ export class EventHandlerStore extends PuterStore {
         return result;
     }
 
+    /**
+     * The scopes one user holds, one row per app, keyset-paginated on
+     * `app_uid`. `total` counts every scope the user holds (or the one app's,
+     * when `appUid` narrows it).
+     */
+    async listUserScopesForUser(
+        userId: number,
+        opts: ListEventsWorkersOptions = {},
+    ): Promise<PageResult<UserScopeSummary>> {
+        const limit = Math.min(
+            Math.max(
+                1,
+                Math.floor(opts.limit ?? EVENTS_WORKERS_LIST_DEFAULT_LIMIT),
+            ),
+            EVENTS_WORKERS_LIST_LIMIT_CAP,
+        );
+        const after = decodeCursor(opts.cursor)?.appUid;
+
+        const where = [`\`${USER_TABLE}\`.\`holder_user_id\` = ?`];
+        const params: unknown[] = [userId];
+        if (typeof opts.appUid === 'string') {
+            where.push(`\`${USER_TABLE}\`.\`app_uid\` = ?`);
+            params.push(opts.appUid);
+        }
+        const totalRows = await this.clients.db.read(
+            'SELECT COUNT(*) AS `total` FROM (' +
+                `SELECT \`${USER_TABLE}\`.\`app_uid\` FROM \`${USER_TABLE}\` ` +
+                `WHERE ${where.join(' AND ')} GROUP BY \`${USER_TABLE}\`.\`app_uid\`` +
+                ') AS `counted`',
+            params,
+        );
+        if (typeof after === 'string') {
+            where.push(`\`${USER_TABLE}\`.\`app_uid\` > ?`);
+            params.push(after);
+        }
+
+        const rows = await this.clients.db.read(
+            `SELECT \`${USER_TABLE}\`.\`app_uid\` AS \`app_uid\`, ` +
+                `\`${APP_TABLE}\`.\`name\` AS \`app_name\`, ` +
+                `\`${APP_TABLE}\`.\`title\` AS \`app_title\`, ` +
+                'COUNT(*) AS `handler_count`, ' +
+                `MIN(\`${USER_TABLE}\`.\`created_at\`) AS \`created_at\`, ` +
+                `MAX(\`${USER_TABLE}\`.\`updated_at\`) AS \`updated_at\` ` +
+                `FROM \`${USER_TABLE}\` ` +
+                `LEFT JOIN \`${APP_TABLE}\` ON \`${APP_TABLE}\`.\`uid\` = \`${USER_TABLE}\`.\`app_uid\` ` +
+                `WHERE ${where.join(' AND ')} ` +
+                `GROUP BY \`${USER_TABLE}\`.\`app_uid\`, \`${APP_TABLE}\`.\`name\`, \`${APP_TABLE}\`.\`title\` ` +
+                `ORDER BY \`${USER_TABLE}\`.\`app_uid\` LIMIT ?`,
+            [...params, limit + 1],
+        );
+
+        const page = keysetPage(rows, limit, undefined, (last) => ({
+            appUid: last.app_uid,
+        }));
+        const result: PageResult<UserScopeSummary> = {
+            items: page.rows.map((row) => ({
+                appUid: String(row.app_uid),
+                appName: row.app_name == null ? '' : String(row.app_name),
+                appTitle: row.app_title == null ? '' : String(row.app_title),
+                handlerCount: Number(row.handler_count) || 0,
+                createdAt: Number(row.created_at) || 0,
+                updatedAt: Number(row.updated_at) || 0,
+            })),
+            total: Number(totalRows[0]?.total ?? 0),
+        };
+        if (page.cursor) result.cursor = page.cursor;
+        return result;
+    }
+
+    /**
+     * The user scopes one app holds, ordered by holder. A batch scan: it
+     * carries no `total`, to spare a COUNT per page. `cursor` is null at the
+     * end.
+     */
+    async listUserScopes(
+        appUid: string,
+        opts: { limit?: number; cursor?: string } = {},
+    ): Promise<{ items: AppUserScope[]; cursor: string | null }> {
+        const limit = Math.min(
+            Math.max(1, Math.floor(opts.limit ?? USER_SCOPES_PAGE_DEFAULT)),
+            USER_SCOPES_PAGE_CAP,
+        );
+        const after = Number(decodeCursor(opts.cursor)?.holder);
+        const rows = await this.clients.db.read(
+            'SELECT `holder_user_id`, COUNT(*) AS `handler_count`, ' +
+                'MIN(`created_at`) AS `created_at`, ' +
+                'MAX(`updated_at`) AS `updated_at` ' +
+                `FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? AND \`holder_user_id\` > ? ` +
+                'GROUP BY `holder_user_id` ORDER BY `holder_user_id` LIMIT ?',
+            [appUid, Number.isFinite(after) ? after : 0, limit + 1],
+        );
+        const page = keysetPage(rows, limit, undefined, (last) => ({
+            holder: Number(last.holder_user_id),
+        }));
+        return {
+            items: page.rows.map((row) => ({
+                holder: Number(row.holder_user_id),
+                handlerCount: Number(row.handler_count) || 0,
+                createdAt: Number(row.created_at) || 0,
+                updatedAt: Number(row.updated_at) || 0,
+            })),
+            cursor: page.cursor ?? null,
+        };
+    }
+
+    /**
+     * One keyset page of user-scope handlers last touched before `olderThan`
+     * (unix seconds) that no subscription row of any state is bound to. A batch
+     * scan: it carries no `total`, to spare a COUNT per page. `cursor` is null
+     * at the end. A candidate may be bound before it is deleted, so delete
+     * through {@link deleteUserHandlerIfUnbound}.
+     */
+    async listUnboundUserHandlers(
+        olderThan: number,
+        opts: { limit?: number; cursor?: string } = {},
+    ): Promise<{ items: UnboundUserHandler[]; cursor: string | null }> {
+        const limit = Math.max(
+            1,
+            Math.floor(opts.limit ?? UNBOUND_HANDLERS_BATCH),
+        );
+        const after = Number(decodeCursor(opts.cursor)?.id);
+        const rows = await this.clients.db.read(
+            'SELECT h.`id`, h.`app_uid`, h.`holder_user_id`, h.`name` ' +
+                `FROM \`${USER_TABLE}\` h ` +
+                'WHERE h.`id` > ? AND h.`updated_at` < ? ' +
+                `AND NOT EXISTS (SELECT 1 FROM \`${SUBSCRIPTION_TABLE}\` s ` +
+                'WHERE s.`holder_user_id` = h.`holder_user_id` ' +
+                'AND s.`app_uid` = h.`app_uid` AND s.`handler_scope` = 1 ' +
+                'AND s.`handler_name` = h.`name`) ' +
+                'ORDER BY h.`id` LIMIT ?',
+            [Number.isFinite(after) ? after : 0, olderThan, limit + 1],
+        );
+        const page = keysetPage(rows, limit, undefined);
+        return {
+            items: page.rows.map((row) => ({
+                id: Number(row.id),
+                appUid: String(row.app_uid),
+                holder: Number(row.holder_user_id),
+                name: String(row.name),
+            })),
+            cursor: page.cursor ?? null,
+        };
+    }
+
     // -- Internals ---------------------------------------------------
 
     async #insert(
-        appUid: string,
+        scope: HandlerScope,
         name: string,
         source: string,
         sourceHash: string,
+        broadcast: boolean,
     ): Promise<EventHandler> {
-        if ((await this.countForApp(appUid)) >= EVENTS_HANDLERS_PER_APP)
-            throw tooManyHandlers();
+        // A user scope's count read also yields when the scope began, so its
+        // age survives the handlers that held it.
+        let scopeSince: number | null = null;
+        if (isUserScope(scope)) {
+            const [state] = await this.clients.db.pread(
+                'SELECT COUNT(*) AS `total`, MIN(`scope_since`) AS `since` ' +
+                    `FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? AND \`holder_user_id\` = ?`,
+                [scope.appUid, scope.holder],
+            );
+            if (Number(state?.total ?? 0) >= handlerCapOf(scope))
+                throw tooManyHandlers(scope);
+            scopeSince = state?.since == null ? null : Number(state.since);
+        } else if ((await this.countForApp(scope)) >= handlerCapOf(scope))
+            throw tooManyHandlers(scope);
 
         const at = nowSeconds();
         try {
-            await this.clients.db.insert(TABLE, {
-                app_uid: appUid,
+            await this.clients.db.insert(tableOf(scope), {
+                app_uid: scope.appUid,
+                ...(isUserScope(scope)
+                    ? {
+                          holder_user_id: scope.holder,
+                          scope_since: scopeSince ?? at,
+                      }
+                    : {}),
                 name,
                 source,
                 source_hash: sourceHash,
@@ -573,9 +1044,25 @@ export class EventHandlerStore extends PuterStore {
             if (isUniqueViolation(err)) throw publishConflict(name);
             throw err;
         }
-        await this.#forgetSet(appUid);
+        // Concurrent creates can all pass the count above. Counted again with
+        // this row in it, each one past the cap takes itself back out — at the
+        // last slot that may refuse more than it had to, never fewer. User
+        // scopes only: reads stop at the cap, so a row past it would be unseen.
+        if (
+            isUserScope(scope) &&
+            (await this.countForApp(scope)) > handlerCapOf(scope)
+        ) {
+            const { where, params } = scopeFilter(scope);
+            await this.clients.db.write(
+                `DELETE FROM \`${tableOf(scope)}\` WHERE ${where} AND \`name\` = ?`,
+                [...params, name],
+            );
+            await this.#forgetSet(scope);
+            throw tooManyHandlers(scope);
+        }
+        await this.#forgetSet(scope, broadcast);
         return {
-            appUid,
+            appUid: scope.appUid,
             name,
             source,
             sourceHash,
@@ -585,23 +1072,25 @@ export class EventHandlerStore extends PuterStore {
     }
 
     async #update(
+        scope: HandlerScope,
         existing: EventHandler,
         source: string,
         sourceHash: string,
     ): Promise<EventHandler> {
         const at = nowSeconds();
+        const { where, params } = scopeFilter(scope);
         // The `source_hash` predicate is what makes the check-then-write a
         // compare-and-set: a publish that raced past the read above updates
         // nothing here, and its caller is told.
         const updated = await this.clients.db.write(
-            `UPDATE \`${TABLE}\` SET \`source\` = ?, \`source_hash\` = ?, ` +
-                '`updated_at` = ? WHERE `app_uid` = ? AND `name` = ? ' +
+            `UPDATE \`${tableOf(scope)}\` SET \`source\` = ?, \`source_hash\` = ?, ` +
+                `\`updated_at\` = ? WHERE ${where} AND \`name\` = ? ` +
                 'AND `source_hash` = ?',
             [
                 source,
                 sourceHash,
                 at,
-                existing.appUid,
+                ...params,
                 existing.name,
                 existing.sourceHash,
             ],
@@ -609,20 +1098,37 @@ export class EventHandlerStore extends PuterStore {
         if (updated?.anyRowsAffected === false)
             throw publishConflict(existing.name);
 
-        await this.#forgetSet(existing.appUid);
+        await this.#forgetSet(scope);
         return { ...existing, source, sourceHash, updatedAt: at };
     }
 
-    /** Drop the delivery path's cached set here and on every peer. */
-    async #forgetSet(appUid: string): Promise<void> {
+    /**
+     * Drop the delivery path's cached set here, and on every peer if
+     * `broadcast`.
+     */
+    async #forgetSet(scope: HandlerScope, broadcast = true): Promise<void> {
         await this.publishCacheKeys({
-            keys: [handlerSetCacheKey(appUid)],
+            keys: [handlerSetCacheKey(scope), replicaSetCacheKey(scope)],
+            broadcast,
+        });
+    }
+
+    /** {@link #forgetSet} for many scopes, as one peer message. */
+    async #forgetSets(scopes: HandlerScope[]): Promise<void> {
+        await this.publishCacheKeys({
+            keys: scopes.flatMap((scope) => [
+                handlerSetCacheKey(scope),
+                replicaSetCacheKey(scope),
+            ]),
             broadcast: true,
         });
     }
 
-    #assertName(name: unknown): string {
-        if (typeof name !== 'string' || !isValidHandlerName(name))
+    #assertName(name: unknown, scope: HandlerScope): string {
+        if (
+            typeof name !== 'string' ||
+            !isValidHandlerName(name, isUserScope(scope))
+        )
             throw invalidName();
         return name;
     }

src/backend/stores/events/EventSubscriptionStore.ts

@@ -155,6 +155,15 @@ const liveRegionsByToken = (
     return live;
 };
 
+/**
+ * Present on every durable row this code caches: the handler scope is stated (a
+ * user row also carries `handlerScope`). A node without user scopes writes rows
+ * without it, and its entries say nothing about scope.
+ */
+const SCOPE_MARKER = 'sc';
+
+const SCOPE_LOOKUP_BATCH = 200;
+
 /** A row we cannot read is a row we cannot deliver against. */
 const parseRow = (raw: string): DispatchSubscription | null => {
     try {
@@ -1015,7 +1024,11 @@ export class EventSubscriptionStore extends PuterStore {
         const write = this.clients.redis.pipeline();
         for (const row of rows) {
             const key = tokenKey(ownerUserId, row.token);
-            write.hset(key, row.subId, JSON.stringify(row));
+            write.hset(
+                key,
+                row.subId,
+                JSON.stringify({ ...row, [SCOPE_MARKER]: 1 }),
+            );
             write.expire(key, DURABLE_CACHE_TTL_SECONDS);
             write.hset(durableMapKey(ownerUserId), row.subId, row.token);
         }
@@ -1269,9 +1282,96 @@ export class EventSubscriptionStore extends PuterStore {
                     subs.push(parsed);
             }
         }
+        await this.#settleScopes(ownerUserId, subs);
         return subs;
     }
 
+    /**
+     * Rows a node without user scopes cached carry no scope marker, so a user
+     * row among them reads as an app row. Those with a handler are resolved
+     * from the table (replica, then primary for what it lacks) and rewritten
+     * with the marker; marked rows cost nothing.
+     */
+    async #settleScopes(
+        ownerUserId: number,
+        rows: DispatchSubscription[],
+    ): Promise<void> {
+        const unmarked: DispatchSubscription[] = [];
+        for (const row of rows) {
+            const marked = (row as unknown as Record<string, unknown>)[
+                SCOPE_MARKER
+            ];
+            delete (row as unknown as Record<string, unknown>)[SCOPE_MARKER];
+            if (marked === undefined && row.durable === true && row.handlerName)
+                unmarked.push(row);
+        }
+        if (unmarked.length === 0) return;
+
+        // A failed lookup throws, as a table read that fails does for the
+        // rest of dispatch: a row of unknown scope must never be returned,
+        // since it would read as an app row.
+        const scopes = new Map<string, number>();
+        const lookup = async (
+            read: 'read' | 'pread',
+            ids: string[],
+        ): Promise<void> => {
+            const found = await this.clients.db[read](
+                'SELECT `sub_id`, `handler_scope` FROM `event_subscriptions` ' +
+                    `WHERE \`sub_id\` IN (${ids.map(() => '?').join(', ')})`,
+                ids,
+            );
+            for (const row of found)
+                scopes.set(String(row.sub_id), Number(row.handler_scope));
+        };
+        for (let i = 0; i < unmarked.length; i += SCOPE_LOOKUP_BATCH) {
+            const ids = unmarked
+                .slice(i, i + SCOPE_LOOKUP_BATCH)
+                .map((row) => row.subId);
+            await lookup('read', ids);
+            const missing = ids.filter((id) => !scopes.has(id));
+            if (missing.length > 0) await lookup('pread', missing);
+        }
+
+        // In neither the replica nor the primary: a ghost entry. Dropped,
+        // as the normal path treats a row that is gone, so it is not
+        // resolved again on every read.
+        const ghosts = unmarked.filter((row) => !scopes.has(row.subId));
+        if (ghosts.length > 0) {
+            const drop = this.clients.redis.pipeline();
+            for (const row of ghosts)
+                drop.hdel(tokenKey(ownerUserId, row.token), row.subId);
+            await drop.exec();
+            for (const row of ghosts) rows.splice(rows.indexOf(row), 1);
+        }
+
+        const resolved = unmarked.filter((row) => scopes.has(row.subId));
+        for (const row of resolved)
+            if (scopes.get(row.subId) === 1)
+                (row as DurableSubscription).handlerScope = 'user';
+
+        // Rewritten only where the entry still stands, so a row dropped
+        // meanwhile is not brought back. Failing to rewrite costs a
+        // repeat lookup, not a wrong answer.
+        try {
+            const check = this.clients.redis.pipeline();
+            for (const row of resolved)
+                check.hexists(tokenKey(ownerUserId, row.token), row.subId);
+            const standing = (await check.exec()) ?? [];
+            const write = this.clients.redis.pipeline();
+            resolved.forEach((row, i) => {
+                if (Number(standing[i]?.[1]) !== 1) return;
+                write.hset(
+                    tokenKey(ownerUserId, row.token),
+                    row.subId,
+                    JSON.stringify({ ...row, [SCOPE_MARKER]: 1 }),
+                );
+            });
+            await write.exec();
+        } catch (err) {
+            console.warn('[events] could not mark cached row scopes', err);
+        }
+    }
+
     /**
      * One pipelined first page per token, which is the whole hash for any token
      * small enough to be stored compactly; only a larger one is scanned
@@ -1316,6 +1416,7 @@ export class EventSubscriptionStore extends PuterStore {
             }
             subs.push(...taken.values());
         }
+        await this.#settleScopes(ownerUserId, subs);
         return subs;
     }
 

src/backend/stores/events/KvShareHandleStore.integration.test.ts

@@ -326,7 +326,7 @@ describe('the notify flag', () => {
 });
 
 describe('live handles by owner and app', () => {
-    it('lists only the owner\'s live handles in that app, oldest first', async () => {
+    it("lists only the owner's live handles in that app, oldest first", async () => {
         const appUid = 'app-owner-listing';
         const first = await store().mint(
             input({ appUid, keyPrefix: 'owned-a:' }),
@@ -502,13 +502,11 @@ describe('the live cache across nodes', () => {
         expect(await store().getLive(row.handle)).toBeNull();
     });
 
-    it('drops an owner\'s cached rows when the owner is suspended, here and on other nodes', async () => {
+    it("drops an owner's cached rows when the owner is suspended, here and on other nodes", async () => {
         const row = await store().mint(
             input({ appUid: 'app-forget', keyPrefix: 'forget-owner:' }),
         );
-        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(
-            false,
-        );
+        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(false);
         const setSuspended = (value: number) =>
             env.server.clients.db.write(
                 'UPDATE `user` SET `suspended` = ? WHERE `id` = ?',
@@ -534,11 +532,7 @@ describe('the live cache across nodes', () => {
                     ([key]) => key === 'outer.pubsub.kv.share.forgetOwner',
                 ),
             ).toEqual([
-                [
-                    'outer.pubsub.kv.share.forgetOwner',
-                    { ownerUserId },
-                    {},
-                ],
+                ['outer.pubsub.kv.share.forgetOwner', { ownerUserId }, {}],
             ]);
 
             // Another node lifting the suspension reaches this one's cache.
@@ -557,13 +551,11 @@ describe('the live cache across nodes', () => {
         }
     });
 
-    it('drops an owner\'s cached rows when the user store suspends or unsuspends them', async () => {
+    it("drops an owner's cached rows when the user store suspends or unsuspends them", async () => {
         const row = await store().mint(
             input({ appUid: 'app-forget', keyPrefix: 'forget-store:' }),
         );
-        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(
-            false,
-        );
+        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(false);
         try {
             await env.server.stores.user.update(ownerUserId, {
                 suspended: true,
@@ -895,21 +887,27 @@ describe('retired handle retention', () => {
         expect(await exists(live)).toBe(true);
     });
 
-    it('purges once per day across nodes, keeping live and recent rows', async () => {
+    it('purges retired handles past the retention, keeping live and recent rows', async () => {
         const kvShare = env.server.services.kvShare;
-        await env.server.clients.redis.del('kvShare:purge:gate');
         const old = await retiredAt('purge-old:', now() - 31 * DAY);
         const recent = await retiredAt('purge-recent:', now() - 29 * DAY);
         const live = await retiredAt('purge-live:', null);
 
-        expect(await kvShare.purgeRevokedHandles()).toBeGreaterThanOrEqual(1);
+        expect(
+            await kvShare.purgeRevokedHandles(new AbortController().signal),
+        ).toBeGreaterThanOrEqual(1);
         expect(await exists(old)).toBe(false);
         expect(await exists(recent)).toBe(true);
         expect(await exists(live)).toBe(true);
+    });
 
-        // Another node checking the same day finds the gate held.
-        const again = await retiredAt('purge-again:', now() - 31 * DAY);
-        expect(await kvShare.purgeRevokedHandles()).toBeNull();
-        expect(await exists(again)).toBe(true);
+    it('stops purging between batches once its signal aborts', async () => {
+        const kvShare = env.server.services.kvShare;
+        const old = await retiredAt('purge-aborted:', now() - 31 * DAY);
+        const controller = new AbortController();
+        controller.abort();
+
+        expect(await kvShare.purgeRevokedHandles(controller.signal)).toBe(0);
+        expect(await exists(old)).toBe(true);
     });
 });

src/backend/stores/events/PendingDeliveryStore.ts

@@ -18,7 +18,10 @@
  */
 
 import { randomUUID } from 'node:crypto';
-import { EVENTS_INVOKE_TIMEOUT_MS } from '../../clients/events/EventsWorkerInvokerClient.js';
+import {
+    EVENTS_INVOKE_TIMEOUT_MS,
+    EVENTS_WORKER_LEASE_FACTOR,
+} from '../../clients/events/EventsWorkerInvokerClient.js';
 import {
     EVENTS_FAILURE_COUNTER_TTL_MS,
     EVENTS_PENDING_DELIVERIES_PER_SUBSCRIPTION,
@@ -316,7 +319,7 @@ export class PendingDeliveryStore extends PuterStore {
             typeof configured === 'number' && configured > 0
                 ? configured
                 : EVENTS_INVOKE_TIMEOUT_MS;
-        return invokeTimeoutMs * 2;
+        return invokeTimeoutMs * EVENTS_WORKER_LEASE_FACTOR;
     }
 
     // -- Writes ------------------------------------------------------

src/backend/stores/events/columnWidths.ts

@@ -19,8 +19,11 @@
 
 import { HttpError } from '../../core/http/HttpError.js';
 
-/** How wide one column is, and the name a caller knows the value by. */
-interface ColumnWidth {
+/**
+ * How wide one column is, and the name a caller knows the value by. Called by
+ * extensions.
+ */
+export interface ColumnWidth {
     /** Column in the migrations, so a schema change is greppable from here. */
     column: string;
     max: number;

src/backend/stores/events/types.ts

@@ -29,6 +29,12 @@ import type { FsOp } from '../../services/events/subjects.js';
  * is declared once here rather than per store.
  */
 
+/**
+ * Which handler set a subscription's `handlerName` refers to: the app's own
+ * (`'app'`) or the holder's scope of that app (`'user'`). Stored as 0 and 1.
+ */
+export type HandlerScopeKind = 'app' | 'user';
+
 /**
  * Whether a row's events also run the app's published handler in its events
  * worker. Only an app has a worker, and only a named handler can be run.
@@ -38,6 +44,20 @@ export const runsInWorker = (row: {
     handlerName?: string | null;
 }): boolean => row.appUid !== null && !!row.handlerName;
 
+/**
+ * Token, and subject, of the subscription row a recurring or `once` schedule
+ * owns. Those rows carry the schedule's lifecycle (suspend, resume, reap) and
+ * are kept out of dispatch, the durable quotas and the durable listing. Called
+ * by extensions.
+ */
+export const SCHEDULE_TOKEN_PREFIX = 'schedule:';
+
+export const scheduleToken = (scheduleUid: string): string =>
+    `${SCHEDULE_TOKEN_PREFIX}${scheduleUid}`;
+
+export const isScheduleRow = (row: { token: string }): boolean =>
+    row.token.startsWith(SCHEDULE_TOKEN_PREFIX);
+
 /**
  * What a row's delivery re-check runs against. An access mode for a filesystem
  * row, which composes with its anchor uid; the whole grant string for a row on
@@ -83,6 +103,8 @@ export interface DispatchSubscription {
     durable?: true;
     delivery?: DeliveryClass;
     handlerName?: string | null;
+    /** Durable rows only: absent means `'app'`. */
+    handlerScope?: HandlerScopeKind;
     /** Durable rows only: handed to the handler, and read nowhere else. */
     context?: string | null;
 }

src/backend/stores/notification/NotificationStore.js

@@ -33,6 +33,9 @@ import { PuterStore } from '../types';
  * }} NotificationInsert
  */
 
+/** How long a probe that found no `send_key` column stands. */
+const SEND_KEY_REPROBE_MS = 60_000;
+
 /**
  * Fixed width and compared byte for byte on every engine, whatever `group`
  * holds.
@@ -75,6 +78,15 @@ const sliceWhere = (userId, { audience, appUid, excludeTypes }) => {
 };
 
 export class NotificationStore extends PuterStore {
+    #sendKeyPresent = false;
+    #sendKeyProbedAt = Number.NEGATIVE_INFINITY;
+    /** @type {Promise<void> | null} */
+    #sendKeyProbe = null;
+
+    async onServerStart() {
+        await this.#hasSendKey();
+    }
+
     // -- Reads --------------------------------------------------------
 
     /**
@@ -96,6 +108,20 @@ export class NotificationStore extends PuterStore {
         return this.#normalizeRow(rows[0]) ?? null;
     }
 
+    /**
+     * The row a delayed send wrote or last replaced, from the primary. Called
+     * by extensions.
+     *
+     * @param {string} sendKey
+     */
+    async getBySendKey(sendKey) {
+        const rows = await this.clients.db.pread(
+            'SELECT * FROM `notification` WHERE `send_key` = ? LIMIT 1',
+            [sendKey],
+        );
+        return this.#normalizeRow(rows[0]) ?? null;
+    }
+
     /**
      * The newest row `appUid` wrote for the user under `group`, or null when it
      * was shown or acknowledged: an older row is never picked up in its place.
@@ -289,6 +315,57 @@ export class NotificationStore extends PuterStore {
         return rows.map((r) => this.#normalizeRow(r));
     }
 
+    /**
+     * The newest row id in a mailbox slice, or null when it has none, from the
+     * primary. Scoped like `listScoped`; rows of every type count.
+     *
+     * @param {number} userId
+     * @param {{ audience: string; appUid?: string | null }} scope
+     * @returns {Promise<number | null>}
+     */
+    async latestScopedId(userId, { audience, appUid }) {
+        const { where, params } = sliceWhere(userId, {
+            audience,
+            appUid,
+            excludeTypes: [],
+        });
+        const [row] = await this.clients.db.pread(
+            `SELECT \`id\` FROM \`notification\`
+             WHERE ${where.join(' AND ')}
+             ORDER BY \`id\` DESC
+             LIMIT 1`,
+            params,
+        );
+        return row ? Number(row.id) : null;
+    }
+
+    /**
+     * The latest replacement in a mailbox slice by (`replaced_at`, `id`), or
+     * null when none was replaced, from the primary. Scoped like `listScoped`.
+     *
+     * @param {number} userId
+     * @param {{ audience: string; appUid?: string | null }} scope
+     * @returns {Promise<{ replacedAt: number; id: number } | null>}
+     */
+    async latestScopedReplacement(userId, { audience, appUid }) {
+        const { where, params } = sliceWhere(userId, {
+            audience,
+            appUid,
+            excludeTypes: [],
+        });
+        where.push('`replaced_at` IS NOT NULL');
+        const [row] = await this.clients.db.pread(
+            `SELECT \`replaced_at\`, \`id\` FROM \`notification\`
+             WHERE ${where.join(' AND ')}
+             ORDER BY \`replaced_at\` DESC, \`id\` DESC
+             LIMIT 1`,
+            params,
+        );
+        return row
+            ? { replacedAt: Number(row.replaced_at), id: Number(row.id) }
+            : null;
+    }
+
     // -- Writes -------------------------------------------------------
 
     /**
@@ -341,6 +418,7 @@ export class NotificationStore extends PuterStore {
             type,
             group_key: key,
             replaced_at: null,
+            send_key: null,
         });
     }
 
@@ -354,6 +432,95 @@ export class NotificationStore extends PuterStore {
         return (await this.create(args)).uid;
     }
 
+    /**
+     * The statements that write a delayed send's notification, for the caller's
+     * transaction. Each runs only while `guard` holds. With `replaceUid` the
+     * open row is replaced in place and takes `sendKey`; the insert, whose uid
+     * is `sendKey`, then runs only if that row was shown or dismissed in
+     * between. Called by extensions.
+     *
+     * @param {{
+     *     sendKey: string;
+     *     userId: number;
+     *     value: unknown;
+     *     type: string;
+     *     audience: string;
+     *     appUid: string | null;
+     *     group?: string | null;
+     *     replaceUid?: string | null;
+     *     at: number;
+     * }} send
+     *   `at` is epoch ms, the row's time either way.
+     * @param {{ table: string; where: string; values: unknown[] }} guard
+     * @returns {{ statement: string; values: unknown[] }[]}
+     */
+    sendEntries(send, guard) {
+        const db = this.clients.db;
+        const serialized = JSON.stringify(send.value ?? {});
+        const seconds = Math.floor(send.at / 1000);
+        const fence = `SELECT 1 FROM \`${guard.table}\` WHERE ${guard.where}`;
+        const entries = [];
+        if (send.replaceUid) {
+            entries.push({
+                statement:
+                    'UPDATE `notification` SET `value` = ?, `shown` = NULL, ' +
+                    `\`created_at\` = ${db.timestampFromUnixSql()}, ` +
+                    '`replaced_at` = ?, `send_key` = ? WHERE `uid` = ? ' +
+                    'AND `user_id` = ? AND `shown` IS NULL ' +
+                    `AND \`acknowledged\` IS NULL AND EXISTS (${fence})`,
+                values: [
+                    serialized,
+                    seconds,
+                    send.at,
+                    send.sendKey,
+                    send.replaceUid,
+                    send.userId,
+                    ...guard.values,
+                ],
+            });
+        }
+        // Already written: the new row's uid is the key, or a replaced row
+        // carries it. Point lookups on `uid`, since a miss on the mostly-NULL
+        // `send_key` index gap-locks where every plain insert goes.
+        const written = send.replaceUid
+            ? {
+                  sql:
+                      'NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ?) ' +
+                      'AND NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ? AND `send_key` = ?)',
+                  values: [send.sendKey, send.replaceUid, send.sendKey],
+              }
+            : {
+                  sql: 'NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ?)',
+                  values: [send.sendKey],
+              };
+        // Postgres can't type a bare parameter in a SELECT list.
+        const param = (type) =>
+            db.case({ postgres: `CAST(? AS ${type})`, otherwise: '?' });
+        entries.push({
+            statement:
+                'INSERT INTO `notification` (`uid`, `user_id`, `value`, `type`, ' +
+                '`audience`, `app_uid`, `group_key`, `created_at`, `send_key`) ' +
+                `SELECT ${param('TEXT')}, ${param('INTEGER')}, ${param('JSONB')}, ` +
+                `${param('TEXT')}, ${param('TEXT')}, ${param('TEXT')}, ` +
+                `${param('TEXT')}, ${db.timestampFromUnixSql()}, ${param('TEXT')} ` +
+                `FROM \`${guard.table}\` WHERE ${guard.where} AND ${written.sql}`,
+            values: [
+                send.sendKey,
+                send.userId,
+                serialized,
+                send.type,
+                send.audience,
+                send.appUid,
+                groupKey(send.group),
+                seconds,
+                send.sendKey,
+                ...guard.values,
+                ...written.values,
+            ],
+        });
+        return entries;
+    }
+
     /**
      * Rewrite a notification the recipient hasn't dismissed. `shown` is cleared
      * with it, so changed wording goes out again on their next connect —
@@ -362,6 +529,10 @@ export class NotificationStore extends PuterStore {
      * `onlyUnseen` also skips a row already shown. `replacedAt`, in epoch ms,
      * records the rewrite as a replacement and dates the row to it.
      *
+     * `send_key` is cleared with it whenever the column exists, on every node:
+     * the content is no longer what a delayed send wrote, so its recovery must
+     * not deliver it under that send's key.
+     *
      * False when no such row exists, so callers can fall back to a fresh
      * notification instead of dropping what they were reporting.
      *
@@ -378,6 +549,7 @@ export class NotificationStore extends PuterStore {
         const serialized =
             typeof value === 'string' ? value : JSON.stringify(value ?? {});
         const set = ['`value` = ?', '`shown` = NULL'];
+        if (await this.#hasSendKey()) set.push('`send_key` = NULL');
         const params = [serialized];
         if (replacedAt !== undefined) {
             set.push(
@@ -478,6 +650,45 @@ export class NotificationStore extends PuterStore {
 
     // -- Internals ----------------------------------------------------
 
+    /**
+     * Whether `notification.send_key` exists, asked of the primary. Present is
+     * kept for the process; absent is asked again at most once a minute, so a
+     * node started before the column was added clears it soon after.
+     *
+     * @returns {Promise<boolean>}
+     */
+    async #hasSendKey() {
+        if (this.#sendKeyPresent) return true;
+        if (Date.now() - this.#sendKeyProbedAt < SEND_KEY_REPROBE_MS)
+            return false;
+        if (this.#sendKeyProbe) {
+            await this.#sendKeyProbe;
+            return this.#sendKeyPresent;
+        }
+        const probe = this.#probeSendKey();
+        this.#sendKeyProbe = probe;
+        try {
+            await probe;
+        } finally {
+            if (this.#sendKeyProbe === probe) this.#sendKeyProbe = null;
+        }
+        return this.#sendKeyPresent;
+    }
+
+    /** Never rejects: an error reads as no column. */
+    async #probeSendKey() {
+        try {
+            await this.clients.db.pread(
+                'SELECT `send_key` FROM `notification` WHERE 1 = 0',
+                [],
+            );
+            this.#sendKeyPresent = true;
+        } catch {
+            this.#sendKeyPresent = false;
+        }
+        this.#sendKeyProbedAt = Date.now();
+    }
+
     #normalizeRow(row) {
         if (!row) return null;
         if (typeof row.value === 'string') {