Merge remote-tracking branch 'upstream/main' into codex/openai-image-default

Co-Authored-By: Claude Opus 5.5 <[email protected]>

.github/workflows/notify-dev.yaml

@@ -0,0 +1,48 @@
+name: Notify HeyPuter (dev)
+
+on:
+  workflow_dispatch:
+    inputs:
+      puter_ref:
+        description: "puter branch, tag or SHA"
+        type: string
+        default: "main"
+
+permissions:
+  contents: read
+
+jobs:
+  authorize:
+    if: github.repository == 'HeyPuter/puter'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require maintain or admin
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ACTOR: ${{ github.triggering_actor }}
+        run: |
+          ROLE=$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$ACTOR/permission" --jq .role_name)
+          if [ "$ROLE" != "maintain" ] && [ "$ROLE" != "admin" ]; then
+            echo "::error::$ACTOR has role '$ROLE'; maintain or admin required"
+            exit 1
+          fi
+
+  notify:
+    needs: authorize
+    runs-on: ubuntu-latest
+    # Holds HEYPUTER_DISPATCH_TOKEN; limited to main so branch copies of this file can't read it.
+    environment: heyputer-dispatch
+    steps:
+      - name: Trigger heyputer dev deploy
+        env:
+          PUTER_REF: ${{ inputs.puter_ref }}
+          TOKEN: ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}
+        run: |
+          jq -nc --arg ref "$PUTER_REF" \
+            '{event_type: "puter-dev-requested", client_payload: {puter_ref: $ref}}' |
+          curl -fsS -X POST \
+            -H "Authorization: token $TOKEN" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/HeyPuter/heyputer/dispatches \
+            -d @-
+          echo "Requested \`$PUTER_REF\`" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/notify-prod.yaml

@@ -8,6 +8,7 @@ on:
 jobs:
   notify:
     runs-on: ubuntu-latest
+    environment: heyputer-dispatch
     steps:
       - name: Trigger heyputer build
         run: |
@@ -15,4 +16,4 @@ jobs:
             -H "Authorization: token ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}" \
             -H "Accept: application/vnd.github.v3+json" \
             https://api.github.com/repos/HeyPuter/heyputer/dispatches \
-            -d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'
+            -d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'

AGENTS.md

@@ -161,11 +161,38 @@ An overview page answers "what is this API and is it what I need?", and a functi
 
 When you add a page, add it to [src/docs/src/sidebar.js](src/docs/src/sidebar.js) under its area, or it won't show up in the navigation. When you add a runnable example, add its HTML to `src/docs/src/playground/examples/<slug>.html`, register the slug in [src/docs/src/examples.js](src/docs/src/examples.js), and link the code block to it by putting the slug after the language with a semicolon (` ```html;kv-set `).
 
+### Function page layout
+
+A function page opens with one sentence that says what the function does, in its own paragraph, such as "Given a prompt, generate an image using AI." Put any more detail in the paragraphs after it.
+
+After the intro, a function page has these `##` sections, in this order:
+
+1. Syntax
+2. Parameters
+3. Return value
+4. Errors, if the function has any
+5. Sections for specific topics, if the page needs them, with `## Model-specific features` last
+6. Examples
+
+[chat.md](src/docs/src/AI/chat.md) follows this layout.
+
+- **Options as a table.** Document an options object as a table with Option, Type, and Description columns. Give each option its own row, even when two options go together, such as `width` and `height`. Order the rows by how often developers use them, most common first, without group rows. In chat.md that's the core options, then reasoning, tools, response handling, request control, and output.
+- **Common options in the table, model-specific behavior in its own section.** A page whose function works with many models or providers, such as [chat.md](src/docs/src/AI/chat.md) and [txt2img.md](src/docs/src/AI/txt2img.md), splits its options this way:
+  - **The options table** under Parameters lists the options that work with every model, and describes each one the way it works for every model.
+  - **`## Model-specific features`**, the last section before Examples, covers everything else. It has a `###` for each model family or provider, named after it, such as "OpenAI models" or "Cloudflare models". Under it go the options only some models accept, and the common options a model handles differently, such as a narrower range, other allowed values, or an option it ignores.
+  - **Write there only what's different.** A `###` lists an option only when that model or provider adds it or handles it differently. An option that works the same everywhere stays out, even if the provider supports it, and a model or provider with nothing different gets no `###`.
+  - **Model-specific features is the only place for model-specific details.** Every limit, format, allowed value, default, price, or quirk that applies to some models goes there, in that model's or provider's `###`, and nowhere else. The options table and topic sections (such as Image inputs or Video input) describe what's true for every model and link to Model-specific features for the rest. Don't add a table or sentence there that compares providers or repeats a model's limits. Each copy is another source of truth to keep in sync, and copies drift when a model changes.
+  - **Link the two.** Directly under the options table, put `Some models accept additional options or handle these options differently. See [Model-specific features](#model-specific-features).` A row whose option behaves differently on some models can also end with `See [Model-specific features](#model-specific-features).`
+- **A long topic gets its own `##` section.** When explaining an option or feature takes more than a sentence or two, such as function calling or output dimensions, give it a section between Errors and Examples and link to it from the options table. This keeps the reference sections short.
+- **Usage patterns are examples.** A way of using the API, such as stopping a stream, goes in a runnable example that the relevant section links to ("See the [stop a stream example](/playground/ai-chat-stop-stream/)."), not in a section of its own.
+- **Each example has a `###` heading** under `## Examples`, so every example shows in the table of contents.
+
 ### Writing docs
 
 These apply to reference pages and recipes alike.
 
 - **Document the developer-facing side only.** The reader is building on top of puter.js, so cover what affects their code: signatures, defaults, return shapes, errors, limits, and behavior they can observe. Leave out internal workings (which service handles the call, storage layout, caching, retries) unless they change what the developer has to do. [rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) is the exception: it publishes every limit, including timeouts and caps that come from how the backend works.
+- **Document Puter as it runs on puter.com.** The docs are for developers building on production Puter. Leave out behavior that only happens on a self-hosted or differently configured server, such as a provider that isn't set up there.
 - **Describe the current state.** Docs aren't a changelog. Write how the API behaves now, without "now", "no longer", "existing code is unaffected", or other wording that compares it to an older version.
 - **Verify before you document.** Run the calls you describe against a running Puter instead of inferring behavior from the source.
 - **Write plainly.** Use the literal words a developer would say out loud, with contractions and everyday words. No idioms or figurative verbs ("walk away from something shared with you" → "remove your own access", "nothing to page through" → "without pagination"). No formal register ("present only while more pages remain" → "includes a `cursor` if there are more pages to load"). Prefer dev terms such as atomic, concurrent, and insertion order.

doc/pagination.md

@@ -63,6 +63,9 @@ store:
   sealed cursor: the last `id` it returned, and the last `(replaced_at, id)`
   among rows up to that id. Each is its own indexed read, merged into one
   page, so a row replaced after a client read it comes back from that cursor.
+  `from=end` with no cursor answers an empty page whose cursor holds the
+  slice's newest `id` and the slice's latest `(replaced_at, id)` as the
+  replacement mark.
 
 Cursors that carry a sort also pin it: a request that passes a cursor plus a
 conflicting sort is rejected with 400.

package-lock.json

@@ -9555,9 +9555,9 @@
       "license": "Unlicense"
     },
     "node_modules/fast-uri": {
-      "version": "3.1.8",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz",
-      "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==",
+      "version": "3.1.9",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz",
+      "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==",
       "funding": [
         {
           "type": "github",
@@ -9869,9 +9869,9 @@
       "license": "ISC"
     },
     "node_modules/follow-redirects": {
-      "version": "1.16.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
-      "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==",
+      "version": "1.16.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.2.tgz",
+      "integrity": "sha512-/kVP2U8z596Ia0gwFrwyIeo74ta5aVmSeU1FfAUMAmcEP5EP7zydmag3SLiN64NaRKef3lxsq7i/g7ltJQDGow==",
       "funding": [
         {
           "type": "individual",
@@ -14244,10 +14244,13 @@
       }
     },
     "node_modules/postal-mime": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/postal-mime/-/postal-mime-3.0.1.tgz",
-      "integrity": "sha512-N86EnYOxZjW1uHQIgSXnnRdnJz+wxmtDsnk3C4R/9rQ1utj8A2znauXZ3Q3+stCHxG5ckY4/NrO09WiPU6rcdQ==",
-      "license": "MIT-0"
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/postal-mime/-/postal-mime-4.2.0.tgz",
+      "integrity": "sha512-QG5Rh8wY/zjwFF1lKWcBbwWcAWR5jWodmMPL4gMik43XDLebtWoAAPyG0lrE2kYYZ5IHTASrepGuGO0YIwNKAg==",
+      "license": "MIT-0",
+      "engines": {
+        "node": ">=20"
+      }
     },
     "node_modules/postcss": {
       "version": "8.5.26",
@@ -14525,9 +14528,9 @@
       "link": true
     },
     "node_modules/qs": {
-      "version": "6.16.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz",
-      "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==",
+      "version": "6.17.1",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.17.1.tgz",
+      "integrity": "sha512-jd3COkaKVgGUW+ZHyEHCG9bH7RoaaiWuMgzsIBjUMVjLAaWGWpoUAVwljVvBHxRu/2qEHnadLGZrkutYUBPQJA==",
       "license": "BSD-3-Clause",
       "dependencies": {
         "es-define-property": "^1.0.1",
@@ -15996,9 +15999,9 @@
       }
     },
     "node_modules/systeminformation": {
-      "version": "5.33.1",
-      "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.33.1.tgz",
-      "integrity": "sha512-DEN6ICHk3Tk0Uf/hrAHh7xlt7iL5CJFBtPZinA0H62DrGG/KPKqq/Nzj6lCXPS4Ay/sf/14zNnk9LpqKzBIc+w==",
+      "version": "5.33.19",
+      "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.33.19.tgz",
+      "integrity": "sha512-Zm7wGeCG7PqfvzegMJIzrxICopWqYCeLATVmvxOOci1isLcNVKgY4Fr0hhYQzvc7cgtK3377jQVChTFAnznOiA==",
       "license": "MIT",
       "os": [
         "darwin",
@@ -17999,16 +18002,6 @@
         }
       }
     },
-    "node_modules/wrangler/node_modules/undici": {
-      "version": "7.29.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz",
-      "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=20.18.1"
-      }
-    },
     "node_modules/wrangler/node_modules/workerd": {
       "version": "1.20260915.1",
       "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260915.1.tgz",
@@ -18311,7 +18304,7 @@
         "openai": "^6.34.0",
         "otpauth": "^9.5.2",
         "pg": "^8.21.0",
-        "postal-mime": "^3.0.1",
+        "postal-mime": "^4.2.0",
         "replicate": "^1.0.0",
         "sharp": "^0.35.5",
         "smtp-server": "^3.19.24",
@@ -19130,7 +19123,7 @@
         "@heyputer/kv.js": "^0.2.1",
         "open": "^10.2.0",
         "path-browserify": "1.0.1",
-        "postal-mime": "^3.0.1",
+        "postal-mime": "^4.2.0",
         "socket.io-client": "^4.8.4"
       },
       "devDependencies": {

src/backend/clients/database/DatabaseClient.test.ts

@@ -285,6 +285,18 @@ describe('AbstractDatabaseClient — SQL generation', () => {
         );
     });
 
+    it('reads the database clock as epoch milliseconds in each dialect', () => {
+        expect(sqlite().nowMsExpr()).toBe(
+            "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+        );
+        expect(mysql().nowMsExpr()).toBe(
+            "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        );
+        expect(postgres().nowMsExpr()).toBe(
+            'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+        );
+    });
+
     it('describes a stored timestamp the way each driver reads it back', () => {
         const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
         expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');
@@ -306,3 +318,13 @@ describe('AbstractDatabaseClient — SQL generation', () => {
         );
     });
 });
+
+
+it('builds an insert for batchWrite and shares it with insert', async () => {
+    const client = new FixtureDatabaseClient([]);
+    const entry = client.insertEntry('widget', { name: "reader's", enabled: true });
+    expect(entry.values).toEqual(["reader's", true]);
+    expect(entry.statement).toContain('VALUES (?, ?)');
+    await client.insert('widget', { name: "reader's", enabled: true });
+    expect(client.writes).toEqual([{ query: entry.statement, params: entry.values }]);
+});

src/backend/clients/database/DatabaseClient.ts

@@ -110,14 +110,23 @@ export class AbstractDatabaseClient extends PuterClient {
         tableName: string,
         data: Record<string, unknown>,
     ): Promise<WriteResult> {
+        const entry = this.insertEntry(tableName, data);
+        return this.write(
+            entry.statement + this.returningIdClause(),
+            entry.values,
+        );
+    }
+
+    /** An insert that can join other writes in a transaction. */
+    insertEntry(tableName: string, data: Record<string, unknown>): BatchEntry {
         const cols = Object.keys(data);
-        const values = Object.values(data);
-        const sql =
-            `INSERT INTO ${this.quoteIdentifier(tableName)} ` +
-            `(${cols.map((c) => this.quoteIdentifier(c)).join(', ')}) ` +
-            `VALUES (${cols.map(() => '?').join(', ')})` +
-            this.returningIdClause();
-        return this.write(sql, values);
+        return {
+            statement:
+                `INSERT INTO ${this.quoteIdentifier(tableName)} ` +
+                `(${cols.map((c) => this.quoteIdentifier(c)).join(', ')}) ` +
+                `VALUES (${cols.map(() => '?').join(', ')})`,
+            values: Object.values(data),
+        };
     }
 
     /** Whether `read()` goes to a separate replica rather than the primary. */
@@ -401,6 +410,21 @@ export class AbstractDatabaseClient extends PuterClient {
         });
     }
 
+    /**
+     * The database's own clock as epoch milliseconds, so nodes whose clocks
+     * disagree still agree on it. The mysql form ignores the session time zone,
+     * so a DST change can't shift it.
+     */
+    nowMsExpr(): string {
+        return this.case({
+            sqlite: "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+            postgres:
+                'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+            otherwise:
+                "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        });
+    }
+
     /**
      * What a read of a timestamp column holding `seconds` returns: sqlite gives
      * back the UTC text it stored, mysql2 and pg a `Date` read in node's time

src/backend/clients/database/SqliteDatabaseClient.test.ts

@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
 import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
 
 /** Highest schema version the migration table can reach. */
-const CURRENT_SCHEMA_VERSION = 95;
+const CURRENT_SCHEMA_VERSION = 98;
 
 /**
  * These suites migrate real files on disk. Idle they finish in well under a
@@ -724,6 +724,85 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
         ]);
     });
 
+    it('keeps notification.send_key unique', async () => {
+        const rows = await client.read('PRAGMA index_list(`notification`)');
+        expect(
+            rows.find((r) => r.name === 'idx_notification_send_key'),
+        ).toMatchObject({ unique: 1 });
+        const columns = await client.read(
+            'PRAGMA index_info(`idx_notification_send_key`)',
+        );
+        expect(columns.map((c) => String(c.name))).toEqual(['send_key']);
+    });
+
+    it('creates event_user_handlers with its unique and holder indexes', async () => {
+        const columns = await client.read(
+            'PRAGMA table_info(`event_user_handlers`)',
+        );
+        expect(columns.map((c) => String(c.name))).toEqual([
+            'id',
+            'app_uid',
+            'holder_user_id',
+            'name',
+            'source',
+            'source_hash',
+            'created_at',
+            'updated_at',
+            'scope_since',
+        ]);
+        const indexes = await client.read(
+            'PRAGMA index_list(`event_user_handlers`)',
+        );
+        expect(
+            indexes.find(
+                (i) => i.name === 'idx_event_user_handlers_app_holder_name',
+            ),
+        ).toMatchObject({ unique: 1 });
+        expect(
+            indexes.find((i) => i.name === 'idx_event_user_handlers_holder_app'),
+        ).toMatchObject({ unique: 0 });
+        const columnsOf = async (index: string) =>
+            (await client.read(`PRAGMA index_info(\`${index}\`)`)).map((c) =>
+                String(c.name),
+            );
+        expect(await columnsOf('idx_event_user_handlers_app_holder_name')).toEqual(
+            ['app_uid', 'holder_user_id', 'name'],
+        );
+        expect(await columnsOf('idx_event_user_handlers_holder_app')).toEqual([
+            'holder_user_id',
+            'app_uid',
+        ]);
+    });
+
+    it('leaves event_handlers as it was', async () => {
+        const columns = await client.read('PRAGMA table_info(`event_handlers`)');
+        expect(columns.map((c) => String(c.name))).toEqual([
+            'id',
+            'app_uid',
+            'name',
+            'source',
+            'source_hash',
+            'created_at',
+            'updated_at',
+        ]);
+    });
+
+    it('adds event_subscriptions.handler_scope, app scope for existing rows', async () => {
+        const columns = await client.read(
+            'PRAGMA table_info(`event_subscriptions`)',
+        );
+        expect(columns.find((c) => c.name === 'handler_scope')).toMatchObject({
+            notnull: 1,
+            dflt_value: '0',
+        });
+        const indexes = await client.read(
+            'PRAGMA index_list(`event_subscriptions`)',
+        );
+        expect(indexes.map((i) => String(i.name))).toContain(
+            'idx_event_subscriptions_holder',
+        );
+    });
+
     it('leaves an already-migrated database untouched on a second boot', async () => {
         const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-'));
         const path = join(dir, 'nested', 'puter.sqlite');

src/backend/clients/database/SqliteDatabaseClient.ts

@@ -134,6 +134,9 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
     [92, ['0097_user-to-user-holder-permission-index.sql']],
     [93, ['0098_notification-groups.sql']],
     [94, ['0099_kv-share-handle-notify.sql']],
+    [95, ['0100_notification-send-key.sql']],
+    [96, ['0101_user-scoped-handlers.sql']],
+    [97, ['0102_notification-scope-id-index.sql']],
 ];
 
 export class SqliteDatabaseClient extends AbstractDatabaseClient {

src/backend/clients/database/migrations/mysql/mysql_mig_54.sql

@@ -0,0 +1,43 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`. See sqlite/0100_notification-send-key.sql for the
+-- rationale. The key is a uuid, so ascii and compared byte for byte. No
+-- per-file applied-state tracking, so every step is guarded and a replay does
+-- nothing.
+
+CALL _puter_add_col('notification', 'send_key', '`send_key` char(36) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
+DELIMITER //
+CREATE PROCEDURE _puter_add_notification_send_key_index()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_send_key'
+  ) THEN
+    ALTER TABLE `notification` ADD UNIQUE KEY `idx_notification_send_key`
+      (`send_key`), ALGORITHM=INPLACE, LOCK=NONE;
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_notification_send_key_index();
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;

src/backend/clients/database/migrations/mysql/mysql_mig_55.sql

@@ -0,0 +1,66 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Handlers an app's user published into their own scope, kept apart from
+-- `event_handlers` so code that predates them never reads them. See
+-- sqlite/0101_user-scoped-handlers.sql for the column rationale.
+--
+-- Idempotent: `CREATE TABLE IF NOT EXISTS` with the indexes declared inline,
+-- and a guarded column add. There is no per-file applied-state tracking, so a
+-- replay has to be a no-op.
+
+CREATE TABLE IF NOT EXISTS `event_user_handlers` (
+    `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+    -- Matches `apps`.`uid` exactly, charset included, so an equality against
+    -- one never falls back to a conversion.
+    `app_uid` char(40) CHARACTER SET latin1 COLLATE latin1_swedish_ci NOT NULL,
+    `holder_user_id` int unsigned NOT NULL,
+    `name` varchar(128) NOT NULL,
+    `source` mediumtext NOT NULL,
+    `source_hash` char(64) NOT NULL,
+    `created_at` bigint NOT NULL,
+    `updated_at` bigint NOT NULL,
+    -- When the scope first held a handler; kept across handler churn, so the
+    -- rent sweep sees the scope's age, not its oldest remaining handler's.
+    `scope_since` bigint NOT NULL,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `idx_event_user_handlers_app_holder_name` (`app_uid`, `holder_user_id`, `name`),
+    KEY `idx_event_user_handlers_holder_app` (`holder_user_id`, `app_uid`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- Guarded here rather than through `_puter_add_col`, so the file also applies
+-- on its own to a database that never ran the earlier migrations.
+DROP PROCEDURE IF EXISTS _puter_add_event_subscriptions_handler_scope;
+DELIMITER //
+CREATE PROCEDURE _puter_add_event_subscriptions_handler_scope()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'event_subscriptions'
+      AND COLUMN_NAME = 'handler_scope'
+  ) THEN
+    ALTER TABLE `event_subscriptions`
+      ADD COLUMN `handler_scope` tinyint unsigned NOT NULL DEFAULT 0,
+      ALGORITHM=INSTANT;
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_event_subscriptions_handler_scope();
+
+DROP PROCEDURE IF EXISTS _puter_add_event_subscriptions_handler_scope;

src/backend/clients/database/migrations/mysql/mysql_mig_56.sql

@@ -0,0 +1,41 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification` (`user_id`, `audience`, `app_uid`, `id`). See
+-- sqlite/0102_notification-scope-id-index.sql for the rationale. No per-file
+-- applied-state tracking, so the index is added only when missing and a
+-- replay does nothing.
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_scope_id_index;
+DELIMITER //
+CREATE PROCEDURE _puter_add_notification_scope_id_index()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_scope_id'
+  ) THEN
+    ALTER TABLE `notification` ADD INDEX `idx_notification_scope_id`
+      (`user_id`, `audience`, `app_uid`, `id`), ALGORITHM=INPLACE, LOCK=NONE;
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_notification_scope_id_index();
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_scope_id_index;

src/backend/clients/database/migrations/postgres/postgres_mig_43.sql

@@ -0,0 +1,24 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- notification.send_key. See sqlite/0100_notification-send-key.sql for the
+-- rationale. Idempotent via IF NOT EXISTS.
+
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS send_key char(36) DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_send_key
+    ON notification (send_key);

src/backend/clients/database/migrations/postgres/postgres_mig_44.sql

@@ -0,0 +1,39 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Handlers an app's user published into their own scope. See
+-- sqlite/0101_user-scoped-handlers.sql for the column rationale.
+-- Idempotent via IF NOT EXISTS.
+
+CREATE TABLE IF NOT EXISTS event_user_handlers (
+    id BIGSERIAL PRIMARY KEY,
+    app_uid VARCHAR(40) NOT NULL,
+    holder_user_id INTEGER NOT NULL,
+    name VARCHAR(128) NOT NULL,
+    source TEXT NOT NULL,
+    source_hash VARCHAR(64) NOT NULL,
+    created_at BIGINT NOT NULL,
+    updated_at BIGINT NOT NULL,
+    scope_since BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_event_user_handlers_app_holder_name
+    ON event_user_handlers (app_uid, holder_user_id, name);
+CREATE INDEX IF NOT EXISTS idx_event_user_handlers_holder_app
+    ON event_user_handlers (holder_user_id, app_uid);
+
+ALTER TABLE event_subscriptions ADD COLUMN IF NOT EXISTS handler_scope smallint NOT NULL DEFAULT 0;

src/backend/clients/database/migrations/postgres/postgres_mig_45.sql

@@ -0,0 +1,23 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- notification (user_id, audience, app_uid, id). See
+-- sqlite/0102_notification-scope-id-index.sql for the rationale. Idempotent via
+-- IF NOT EXISTS.
+
+CREATE INDEX IF NOT EXISTS idx_notification_scope_id
+    ON notification (user_id, audience, app_uid, id);

src/backend/clients/database/migrations/sqlite/0100_notification-send-key.sql

@@ -0,0 +1,27 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`: the key of the delayed send (a `deferred`
+-- schedule) that wrote this row or last replaced it, derived from the
+-- schedule and its slot. Unique, so one slot can never write two rows, and a
+-- send that crashed after writing finds its row again by it. Null on every row
+-- a delayed send never touched.
+
+ALTER TABLE `notification` ADD COLUMN `send_key` TEXT DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS `idx_notification_send_key`
+    ON `notification` (`send_key`);

src/backend/clients/database/migrations/sqlite/0101_user-scoped-handlers.sql

@@ -0,0 +1,48 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Handlers an app's user published into their own scope, in a table of their
+-- own so `event_handlers` stays the app owner's set. Same columns as
+-- `event_handlers` (see 0076_event-handlers.sql) plus the holder.
+--
+--   - `holder_user_id` : the user whose scope this is. A name is unique per
+--                        (app, holder), so the same name can live in the app's
+--                        set and in each holder's.
+--   - `(holder_user_id, app_uid)` : a holder's scopes, for listing and for the
+--                        rent count.
+--
+-- `event_subscriptions.handler_scope` says which table a row's `handler_name`
+-- refers to: 0 the app's set, 1 the row holder's scope.
+
+CREATE TABLE IF NOT EXISTS `event_user_handlers` (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "app_uid" TEXT NOT NULL,
+    "holder_user_id" INTEGER NOT NULL,
+    "name" TEXT NOT NULL,
+    "source" TEXT NOT NULL,
+    "source_hash" TEXT NOT NULL,
+    "created_at" INTEGER NOT NULL,    -- unix seconds
+    "updated_at" INTEGER NOT NULL,
+    "scope_since" INTEGER NOT NULL    -- unix seconds; the scope's first handler, kept across churn
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS `idx_event_user_handlers_app_holder_name`
+    ON `event_user_handlers` (`app_uid`, `holder_user_id`, `name`);
+CREATE INDEX IF NOT EXISTS `idx_event_user_handlers_holder_app`
+    ON `event_user_handlers` (`holder_user_id`, `app_uid`);
+
+ALTER TABLE `event_subscriptions` ADD COLUMN `handler_scope` INTEGER NOT NULL DEFAULT 0;

src/backend/clients/database/migrations/sqlite/0102_notification-scope-id-index.sql

@@ -0,0 +1,22 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Index a mailbox slice by row id, so a fetch finds the newest row of a slice,
+-- and pages forward through one, with a seek rather than a scan of the slice.
+
+CREATE INDEX IF NOT EXISTS `idx_notification_scope_id`
+    ON `notification` (`user_id`, `audience`, `app_uid`, `id`);

src/backend/clients/event/types.ts

@@ -200,6 +200,16 @@ export type EventMap = {
     };
 
     // ---- Auth / signup ----
+    /**
+     * Which signup methods are open, emitted via `emitAndWait` before every
+     * signup and whenever the GUI reads the provider list. A listener sets
+     * `oidcOnly` to refuse form and temp signups, leaving OIDC and team
+     * provisioning open; `message` is what a refused form signup is told.
+     */
+    'puter.signup.policy': {
+        oidcOnly: boolean;
+        message: string | null;
+    };
     'puter.signup.validate': {
         allow: boolean;
         email?: string;
@@ -323,6 +333,8 @@ export type EventMap = {
         // the abuse extension cap sends per device (across accounts) so a
         // device farm can't dodge the per-account cap with fresh signups.
         device_fingerprint: string | null;
+        // Browser-signals dispatch id the client sent, or null.
+        dispatch_id: string | null;
         allowed: boolean;
         reason: string | null;
         // With `allowed` false, offer the card fallback in place of SMS. Only
@@ -544,6 +556,14 @@ export type EventMap = {
     'kv.share.retired': { handles: string[]; userId: number };
 
     // ---- Notifications ----
+    /** An app message was acknowledged after its state committed. */
+    'notif.acknowledged': {
+        userId: number;
+        uid: string;
+        type: string;
+        audience: string;
+        appUid: string;
+    };
     /**
      * A notification row exists. Emitted after the insert commits — nothing
      * pushes a uid before there is a row to name — and carries the scope tuple
@@ -577,8 +597,12 @@ export type EventMap = {
         value: Record<string, unknown>;
         /** An earlier notification rewritten in place. */
         updated?: true;
+        /** Replaced an earlier notification in its group. */
+        replaced?: true;
         /** Names this one send, for a receipt asked about it. */
         sendId: string;
+        /** The same `sendId` may be emitted again, as a delayed send is. */
+        replayable?: true;
     };
     /**
      * One notification, addressed at whatever sockets each region holds for the
@@ -662,9 +686,18 @@ export type EventMap = {
     // and going away (1→0, whether by removing the last handler or by the
     // destroy route). `actor.user` is the app's owner, not the caller — a
     // developer session publishing for an app it owns is the common case, but
-    // billing follows ownership.
-    'events.worker.create': { actor: Actor; appUid: string };
-    'events.worker.destroy': { actor: Actor; appUid: string };
+    // billing follows ownership. For `scope: 'user'` the unit is one user's
+    // handlers in one app instead, and `actor.user` is that user.
+    'events.worker.create': {
+        actor: Actor;
+        appUid: string;
+        scope: 'app' | 'user';
+    };
+    'events.worker.destroy': {
+        actor: Actor;
+        appUid: string;
+        scope: 'app' | 'user';
+    };
 
     // ---- Outer / GUI broadcast ----
     'outer.cacheUpdate': {
@@ -950,6 +983,9 @@ export type AiCostFactorEvent = {
  */
 export type LifecyclePhase = 'before' | 'after' | 'error' | 'reject';
 
+/** Status a vetoing `before` listener may answer with. */
+export type LifecycleRejectStatus = 403 | 429;
+
 /**
  * Payload for `driver.<iface>.<method>.<phase>` events.
  *
@@ -979,6 +1015,8 @@ export type DriverMethodLifecycleEvent = {
     allow?: boolean;
     /** Optional human-readable reason surfaced to the caller when vetoed. */
     rejectReason?: string;
+    /** Status for a veto: 403 (default) or 429 for a throttle. */
+    rejectStatus?: LifecycleRejectStatus;
 };
 
 /**
@@ -1013,6 +1051,8 @@ export type RouteLifecycleEvent = {
     allow?: boolean;
     /** Optional human-readable reason surfaced to the caller when vetoed. */
     rejectReason?: string;
+    /** Status for a veto: 403 (default) or 429 for a throttle. */
+    rejectStatus?: LifecycleRejectStatus;
 };
 
 export type EventKey = keyof EventMap & string;

src/backend/clients/events/EventsWorkerInvokerClient.ts

@@ -35,6 +35,7 @@ import { PuterClient } from '../types.js';
  *     x-puter-internal-auth:  <internal secret>
  *     x-puter-events-script:  <script the handler set deploys as>
  *     x-puter-events-app:     <app uid, for the deploy-on-miss callback>
+ *     x-puter-events-holder:  <holder uuid of a user scope; absent for app scope>
  *     x-puter-events-key:     <key derived for that script>
  *     { handler, event, ctx, token }
  *
@@ -43,6 +44,20 @@ import { PuterClient } from '../types.js';
  *
  * On a miss, a handler set here may deploy the script itself and retry once
  * with `x-puter-events-deployed: 1`, skipping the dispatcher's own callback.
+ *
+ * Calls that come due within a few milliseconds of each other go out together:
+ *
+ *     POST <events dispatcher>/invoke-batch
+ *     x-puter-internal-auth:  <internal secret>
+ *     { calls: [{ script, appUid, holder?, key, deployed?, body, waitMs }] }
+ *
+ * The dispatcher runs a few at a time, in order, and streams NDJSON: `{ i,
+ * started: true }` when a call goes to its worker, then `{ i, status, handled?,
+ * error?, retryAfter?, dispatch? }` when it finishes — the same facts the
+ * single route carries in its status and headers. A call's timeout runs from
+ * its `started` line, so time spent queued behind other calls is never the
+ * handler's. A call still queued `waitMs` after the batch arrived is never
+ * started and answers `dispatch: "queue-timeout"`.
  */
 
 /** The one route an events worker answers. */
@@ -51,6 +66,27 @@ export const EVENTS_INVOKE_PATH = '/__events/invoke';
 /** The route the events dispatcher takes an invocation on. */
 export const EVENTS_DISPATCH_PATH = '/invoke';
 
+/** The route it takes several on, answering each as it finishes. */
+export const EVENTS_DISPATCH_BATCH_PATH = '/invoke-batch';
+
+/** Calls one batch carries at most. */
+export const INVOKE_BATCH_SIZE = 60;
+
+/** How long the first call of a batch waits for others to join it. */
+export const INVOKE_BATCH_WINDOW_MS = 5;
+
+/**
+ * How long a dispatcher without the batch route is sent single calls before the
+ * batch route is tried again.
+ */
+const BATCH_UNSUPPORTED_RETRY_MS = 5 * 60_000;
+
+/**
+ * How long past its wait a batched call's `started` line may take to arrive
+ * before the call is given up as never started.
+ */
+const BATCH_START_GRACE_MS = 2_000;
+
 /**
  * Header the dispatcher marks its own failures with. Anything carrying it never
  * reached a handler, so its status says nothing about the delivery.
@@ -92,6 +128,12 @@ const PLATFORM_ERROR_CODES = new Set([
  */
 export const EVENTS_DEPLOYED_HEADER = 'x-puter-events-deployed';
 
+/**
+ * Names the user whose scope a script belongs to, so a miss deploys it from
+ * their handlers. Absent for an app's own scope.
+ */
+export const EVENTS_HOLDER_HEADER = 'x-puter-events-holder';
+
 /**
  * Dispatch-error reasons this backend can resolve itself by deploying, rather
  * than wait on the rehydrate callback finding a backend behind the dispatcher's
@@ -112,21 +154,90 @@ const SELF_DEPLOYABLE_MISS_REASONS = new Set([
  */
 const ALREADY_DEPLOYED_MISS_REASONS = new Set(['deploy-timeout']);
 
+/** Whose call asked for the deploy, for the share of deploys it may take. */
+export interface WorkerMissOptions {
+    lane?: InvokeLane;
+    /**
+     * Holder uuid of the user scope the script belongs to; absent for app
+     * scope.
+     */
+    holder?: string;
+}
+
 /** What a self-deploy attempt did, and to which app/script it applies. */
 export type WorkerMissHandler = (
     appUid: string,
     script: string,
+    options: WorkerMissOptions,
 ) => Promise<'deployed' | string>;
 
+/**
+ * `work`'s result, or null once `ms` has passed. The work keeps running, and a
+ * failure after the cutoff is logged since nothing else would see it.
+ */
+const withinTime = async <T>(
+    work: Promise<T>,
+    ms: number,
+): Promise<T | null> => {
+    let timedOut = false;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const limit = new Promise<null>((resolve) => {
+        timer = setTimeout(
+            () => {
+                timedOut = true;
+                resolve(null);
+            },
+            Math.max(0, ms),
+        );
+    });
+    void (async () => {
+        try {
+            await work;
+        } catch (err) {
+            if (timedOut) console.warn('[events] deploy failed late', err);
+        }
+    })();
+    try {
+        return await Promise.race([work, limit]);
+    } finally {
+        clearTimeout(timer);
+    }
+};
+
 /** How long a handler has to answer before the attempt is abandoned. */
 export const EVENTS_INVOKE_TIMEOUT_MS = 30_000;
 
+/**
+ * How many invoke timeouts a delivery's worker lease lasts. The lease store
+ * reads this, and `invoke()` keeps a whole call, deploy and resend included,
+ * inside it.
+ */
+export const EVENTS_WORKER_LEASE_FACTOR = 2;
+
+/**
+ * Left unspent at the end of the lease, and the least a resend is worth
+ * starting with: below it the call is given up as `unavailable` and retried.
+ */
+export const EVENTS_DEADLINE_MARGIN_MS = 5_000;
+
 /**
  * Connections held open to the dispatcher, so a busy fleet is not
  * re-handshaking. Also how many calls one transport makes at once: a call past
  * that waits for a free one before its invoke timeout starts.
  */
 export const INVOKE_CONNECTIONS = 256;
+
+/**
+ * Of those, how many schedule runs may hold at once, so a burst of schedules
+ * can never take the connections event deliveries need.
+ */
+export const SCHEDULE_INVOKE_CONNECTIONS = 64;
+
+/** Whose call this is, for the connection share it may take. */
+export type InvokeLane = 'delivery' | 'schedule';
+
+/** What a deploy answers when another deploy of the region holds the way. */
+export const DEPLOY_BUSY = 'busy';
 const INVOKE_KEEP_ALIVE_MS = 30_000;
 
 /**
@@ -141,6 +252,8 @@ export interface WorkerInvokeRequest {
     /** Script the app's current handler set deploys as. */
     script: string;
     appUid: string;
+    /** Holder uuid, when the handler lives in a user scope. */
+    holder?: string;
     handler: string;
     /** The subscriber's app session token. */
     token: string;
@@ -154,6 +267,8 @@ export interface WorkerInvokeRequest {
 export interface EventsInvokeCall {
     script: string;
     appUid: string;
+    /** Holder uuid of the user scope the script belongs to. */
+    holder?: string;
     /** Value the script compares against its own invoke-key binding. */
     key: string;
     /** Serialized `{ handler, event, ctx, token }`. */
@@ -161,6 +276,17 @@ export interface EventsInvokeCall {
     timeoutMs: number;
     /** Set on the retry after this backend deployed the script itself. */
     deployed?: boolean;
+    lane?: InvokeLane;
+    /**
+     * Builds `body` again at send time, for a transport that waits for a
+     * connection first, so a send time stamped into it is the real one.
+     */
+    render?: () => string;
+    /**
+     * Rechecked by a transport that waits before sending; false drops the call
+     * with {@link supersededAnswer}.
+     */
+    beforeSend?: () => Promise<boolean>;
 }
 
 /** What one attempt at delivering an invocation came back with. */
@@ -176,6 +302,8 @@ export interface EventsInvokeAnswer {
     timedOut?: boolean;
     /** From a `retry-after` header in seconds. */
     retryAfterMs?: number;
+    /** A deploy this call needed was put off: another held the way. */
+    deployBusy?: boolean;
 }
 
 /**
@@ -198,6 +326,8 @@ export type WorkerInvokeOutcome =
 
 export interface WorkerInvokeResult {
     outcome: WorkerInvokeOutcome;
+    /** A deploy on miss was put off because another one held the way. */
+    deployBusy?: boolean;
     /** The status the handler answered with, or `null` when it never did. */
     status: number | null;
     /** Why it could not answer, when nothing did. */
@@ -206,9 +336,24 @@ export interface WorkerInvokeResult {
     retryAfterMs?: number;
 }
 
+/** How one invocation is sent. */
+export interface WorkerInvokeOptions {
+    lane?: InvokeLane;
+    /** Rechecked before each send, including a resend after deployment. */
+    beforeSend?: () => Promise<boolean>;
+    /** Applied to the event as each attempt is sent. */
+    stampEvent?: (event: unknown) => unknown;
+}
+
 const NO_TRANSPORT =
     'no events dispatcher configured (url and internal secret)';
 
+/** What a call whose `beforeSend` said no answers instead of being sent. */
+const supersededAnswer = (): EventsInvokeAnswer => ({
+    status: null,
+    error: 'invocation superseded',
+});
+
 export class EventsWorkerInvokerClient extends PuterClient {
     /** Set only where invocations do not leave the process — local development. */
     #transport: EventsInvokeTransport | null = null;
@@ -236,7 +381,10 @@ export class EventsWorkerInvokerClient extends PuterClient {
         this.#dispatcher = null;
     }
 
-    async invoke(request: WorkerInvokeRequest): Promise<WorkerInvokeResult> {
+    async invoke(
+        request: WorkerInvokeRequest,
+        options: WorkerInvokeOptions = {},
+    ): Promise<WorkerInvokeResult> {
         const transport = this.#pickTransport();
         if (!transport)
             return {
@@ -245,32 +393,86 @@ export class EventsWorkerInvokerClient extends PuterClient {
                 error: NO_TRANSPORT,
             };
 
-        const call: EventsInvokeCall = {
-            script: request.script,
-            appUid: request.appUid,
-            key: request.key,
-            body: JSON.stringify({
+        const { stampEvent } = options;
+        const render = (): string =>
+            JSON.stringify({
                 handler: request.handler,
-                event: request.event,
+                event: stampEvent ? stampEvent(request.event) : request.event,
                 ctx: request.ctx,
                 token: request.token,
-            }),
-            timeoutMs: this.#timeoutMs(),
+            });
+        const timeoutMs = this.#timeoutMs();
+        // The whole call, deploy and resend included, ends before the lease a
+        // delivery holds would let it be claimed again mid-run.
+        // A short configured timeout gets a proportionally short margin.
+        const margin = Math.min(EVENTS_DEADLINE_MARGIN_MS, timeoutMs / 2);
+        const deadline =
+            Date.now() + timeoutMs * EVENTS_WORKER_LEASE_FACTOR - margin;
+        const send = async (
+            deployed: boolean,
+            sendTimeoutMs = timeoutMs,
+        ): Promise<EventsInvokeAnswer> => {
+            if (options.beforeSend && !(await options.beforeSend()))
+                return supersededAnswer();
+            return transport.send({
+                script: request.script,
+                appUid: request.appUid,
+                ...(request.holder ? { holder: request.holder } : {}),
+                key: request.key,
+                body: render(),
+                timeoutMs: sendTimeoutMs,
+                ...(deployed ? { deployed: true } : {}),
+                ...(options.lane ? { lane: options.lane } : {}),
+                ...(stampEvent ? { render } : {}),
+                ...(options.beforeSend
+                    ? { beforeSend: options.beforeSend }
+                    : {}),
+            });
         };
 
-        let result = await transport.send(call);
-        if (this.#missHandler && selfDeployable(result)) {
-            const outcome = await this.#missHandler(
-                request.appUid,
-                request.script,
+        // A send may wait for a connection for half its timeout before the
+        // run's own timeout starts, so it is given what is left divided by that.
+        const resend = (): Promise<EventsInvokeAnswer> | null => {
+            const remaining = deadline - Date.now();
+            if (remaining < margin) return null;
+            return send(
+                true,
+                Math.floor(
+                    Math.min(
+                        timeoutMs,
+                        remaining / (1 + CONNECTION_WAIT_SHARE),
+                    ),
+                ),
             );
+        };
+        const outOfTime = (): WorkerInvokeResult => ({
+            outcome: 'unavailable',
+            status: null,
+            error: 'out of time to resend after deploy',
+        });
+
+        let result = await send(false);
+        if (this.#missHandler && selfDeployable(result)) {
+            // The deploy may be shared with other callers, so it is left
+            // running when this call runs out of time for it.
+            const outcome = await withinTime(
+                this.#missHandler(request.appUid, request.script, {
+                    lane: options.lane,
+                    ...(request.holder ? { holder: request.holder } : {}),
+                }),
+                deadline - Date.now() - margin,
+            );
+            if (outcome === null) return outOfTime();
             if (outcome !== 'deployed')
                 return {
                     outcome: 'unavailable',
                     status: null,
                     error: `deploy: ${outcome}`,
+                    ...(outcome === DEPLOY_BUSY ? { deployBusy: true } : {}),
                 };
-            result = await transport.send({ ...call, deployed: true });
+            const resent = await resend();
+            if (!resent) return outOfTime();
+            result = resent;
         } else if (
             result.status === null &&
             result.dispatchReason &&
@@ -278,10 +480,19 @@ export class EventsWorkerInvokerClient extends PuterClient {
         ) {
             // No deploy to trigger — just give propagation the one retry
             // it usually needs, without spending an upload on it.
-            result = await transport.send({ ...call, deployed: true });
+            const resent = await resend();
+            if (!resent) return outOfTime();
+            result = resent;
         }
 
         const { status, handled, error, timedOut, retryAfterMs } = result;
+        if (result.deployBusy)
+            return {
+                outcome: 'unavailable',
+                status: null,
+                error,
+                deployBusy: true,
+            };
         // Past the timeout the handler was most likely still running, which
         // is its own doing; any other missing answer never reached it.
         if (status === null)
@@ -313,9 +524,19 @@ export class EventsWorkerInvokerClient extends PuterClient {
         // a misconfiguration worth reporting once per attempt rather than a
         // round trip.
         if (!dispatcherUrl || !internalSecret) return null;
+        const { invokeBatchSize, invokeBatchWindowMs } =
+            this.config.events ?? {};
         this.#dispatcher ??= new DispatcherInvokeTransport(
             dispatcherUrl,
             internalSecret,
+            {
+                ...(invokeBatchSize !== undefined
+                    ? { batchSize: invokeBatchSize }
+                    : {}),
+                ...(invokeBatchWindowMs !== undefined
+                    ? { batchWindowMs: invokeBatchWindowMs }
+                    : {}),
+            },
         );
         return this.#dispatcher;
     }
@@ -328,28 +549,65 @@ export class EventsWorkerInvokerClient extends PuterClient {
     }
 }
 
+/** A call waiting in, or sent with, the current batch. */
+interface BatchedCall {
+    call: EventsInvokeCall;
+    resolve: (answer: EventsInvokeAnswer) => void;
+    /** For a `beforeSend` that threw. */
+    reject: (err: unknown) => void;
+    /** When its wait for a connection began, for what is left of it. */
+    waitingSince: number;
+}
+
+export interface DispatcherInvokeTransportOptions {
+    fetchImpl?: typeof undiciFetch;
+    /** Calls per batch; 1 sends every call on its own. */
+    batchSize?: number;
+    batchWindowMs?: number;
+}
+
 /** Reaches an events worker the only way production can: through the dispatcher. */
 export class DispatcherInvokeTransport implements EventsInvokeTransport {
     readonly #url: string;
     readonly #secret: string;
     readonly #fetchImpl: typeof undiciFetch;
+    readonly #batchSize: number;
+    readonly #batchWindowMs: number;
     #agent: Agent | null = null;
     /** Calls holding a connection slot. */
     #active = 0;
-    /** Calls waiting for one, longest-waiting first. */
-    readonly #waiting = new Set<() => void>();
+    /** Of those, schedule runs. */
+    #scheduleActive = 0;
+    /** Calls waiting for one, longest-waiting first, with their lane. */
+    readonly #waiting = new Map<() => void, InvokeLane>();
+    /** Calls gathered for the next batch. */
+    #batch: BatchedCall[] = [];
+    #batchTimer: ReturnType<typeof setTimeout> | null = null;
+    /** Until when the dispatcher is taken not to have the batch route. */
+    #batchUnsupportedUntil = 0;
 
     constructor(
         dispatcherUrl: string,
         secret: string,
-        options: { fetchImpl?: typeof undiciFetch } = {},
+        options: DispatcherInvokeTransportOptions = {},
     ) {
-        this.#url = dispatcherUrl;
+        // Joined as a string: `new URL(path, base)` resolves an absolute path
+        // against the base's origin, dropping any path prefix it carries.
+        this.#url = dispatcherUrl.replace(/\/$/, '');
         this.#secret = secret;
         this.#fetchImpl = options.fetchImpl ?? undiciFetch;
+        this.#batchSize = Math.max(
+            1,
+            Math.floor(options.batchSize ?? INVOKE_BATCH_SIZE),
+        );
+        this.#batchWindowMs = Math.max(
+            0,
+            options.batchWindowMs ?? INVOKE_BATCH_WINDOW_MS,
+        );
     }
 
     close(): void {
+        this.#flush();
         void this.#agent?.close().catch(() => {});
         this.#agent = null;
     }
@@ -357,29 +615,102 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
     async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
         // No `timedOut`: the handler never got this call, so the wait is not
         // its failure.
-        if (!(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE)))
+        const lane = call.lane ?? 'delivery';
+        const waitingSince = Date.now();
+        if (
+            !(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE, lane))
+        )
             return {
                 status: null,
                 error: 'no free connection to the events dispatcher',
             };
         try {
-            // `new URL(path, base)` resolves an absolute path against the
-            // base's origin, dropping any path prefix `base` already carries.
-            const url = `${this.#url.replace(/\/$/, '')}${EVENTS_DISPATCH_PATH}`;
-            const response = await this.#fetchImpl(url, {
-                method: 'POST',
-                headers: {
-                    'content-type': 'application/json',
-                    'x-puter-internal-auth': this.#secret,
-                    'x-puter-events-script': call.script,
-                    'x-puter-events-app': call.appUid,
-                    'x-puter-events-key': call.key,
-                    ...(call.deployed ? { [EVENTS_DEPLOYED_HEADER]: '1' } : {}),
-                },
-                body: call.body,
-                signal: AbortSignal.timeout(call.timeoutMs),
-                dispatcher: this.#dispatcherFor(),
+            if (this.#batchSize === 1) {
+                // The wait for a connection may have outlived the call.
+                if (call.beforeSend && !(await call.beforeSend()))
+                    return supersededAnswer();
+                return await this.#sendOne(call);
+            }
+            return await new Promise<EventsInvokeAnswer>((resolve, reject) => {
+                this.#batch.push({ call, resolve, reject, waitingSince });
+                if (this.#batch.length >= this.#batchSize) this.#flush();
+                else
+                    this.#batchTimer ??= setTimeout(
+                        () => this.#flush(),
+                        this.#batchWindowMs,
+                    );
             });
+        } finally {
+            this.#release(lane);
+        }
+    }
+
+    /** Send what has gathered: alone when it is one call, else as a batch. */
+    #flush(): void {
+        if (this.#batchTimer) clearTimeout(this.#batchTimer);
+        this.#batchTimer = null;
+        const batch = this.#batch;
+        this.#batch = [];
+        if (batch.length === 0) return;
+        if (batch.some(({ call }) => call.beforeSend))
+            void this.#sendStillCurrent(batch);
+        else this.#sendGathered(batch);
+    }
+
+    #sendGathered(batch: BatchedCall[]): void {
+        if (batch.length === 1 || Date.now() < this.#batchUnsupportedUntil) {
+            for (const { call, resolve } of batch)
+                void this.#sendOne(call).then(resolve);
+            return;
+        }
+        void this.#sendBatch(batch);
+    }
+
+    /** Drops calls superseded while they were gathered, then sends the rest. */
+    async #sendStillCurrent(batch: BatchedCall[]): Promise<void> {
+        const checked = await Promise.all(
+            batch.map(async (entry) => {
+                if (!entry.call.beforeSend) return entry;
+                try {
+                    if (await entry.call.beforeSend()) return entry;
+                    entry.resolve(supersededAnswer());
+                } catch (err) {
+                    entry.reject(err);
+                }
+                return null;
+            }),
+        );
+        const current = checked.filter(
+            (entry): entry is BatchedCall => entry !== null,
+        );
+        if (current.length > 0) this.#sendGathered(current);
+    }
+
+    /** One call on the single route. */
+    async #sendOne(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                        'x-puter-events-script': call.script,
+                        'x-puter-events-app': call.appUid,
+                        ...(call.holder
+                            ? { [EVENTS_HOLDER_HEADER]: call.holder }
+                            : {}),
+                        'x-puter-events-key': call.key,
+                        ...(call.deployed
+                            ? { [EVENTS_DEPLOYED_HEADER]: '1' }
+                            : {}),
+                    },
+                    body: call.render ? call.render() : call.body,
+                    signal: AbortSignal.timeout(call.timeoutMs),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
             // The body is not read: a handler answers with a status, and
             // whatever else it writes is its own business.
             void response.body?.cancel().catch(() => {});
@@ -390,24 +721,198 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
                 EVENTS_DISPATCH_ERROR_HEADER,
             );
             if (dispatchError !== null)
-                return {
-                    status: null,
-                    error: `dispatcher: ${dispatchError} (${response.status})`,
-                    dispatchReason: dispatchError,
-                };
+                return dispatchAnswer(dispatchError, response.status);
 
             return answerFrom(response);
         } catch (err) {
             return failedAnswer(err);
-        } finally {
-            this.#release();
         }
     }
 
+    /**
+     * Several calls in one request. A call may wait in the dispatcher's queue
+     * for whatever is left of its connection wait, and its timeout runs from
+     * its `started` line; the request itself is dropped once every call has an
+     * answer.
+     */
+    async #sendBatch(batch: BatchedCall[]): Promise<void> {
+        const hangUp = new AbortController();
+        let open = batch.length;
+        const answered = batch.map(() => false);
+        const timers: ReturnType<typeof setTimeout>[] = [];
+        const finish = (i: number, done: (entry: BatchedCall) => void) => {
+            if (answered[i]) return;
+            answered[i] = true;
+            clearTimeout(timers[i]);
+            done(batch[i]);
+            if (--open === 0) hangUp.abort();
+        };
+        const settle = (i: number, answer: EventsInvokeAnswer): void =>
+            finish(i, (entry) => entry.resolve(answer));
+        const settleRest = (answer: EventsInvokeAnswer): void => {
+            for (let i = 0; i < batch.length; i++) settle(i, answer);
+        };
+
+        const now = Date.now();
+        const waits = batch.map(({ call, waitingSince }) =>
+            Math.max(
+                0,
+                Math.floor(
+                    call.timeoutMs * CONNECTION_WAIT_SHARE -
+                        (now - waitingSince),
+                ),
+            ),
+        );
+        const body = JSON.stringify({
+            calls: batch.map(({ call }, i) => ({
+                script: call.script,
+                appUid: call.appUid,
+                ...(call.holder ? { holder: call.holder } : {}),
+                key: call.key,
+                ...(call.deployed ? { deployed: true } : {}),
+                body: call.render ? call.render() : call.body,
+                waitMs: waits[i],
+            })),
+        });
+        const arm = (
+            i: number,
+            ms: number,
+            answer: () => EventsInvokeAnswer,
+        ) => {
+            clearTimeout(timers[i]);
+            timers[i] = setTimeout(() => settle(i, answer()), ms);
+            timers[i].unref?.();
+        };
+        // No `timedOut`: a call that never started never reached a handler.
+        batch.forEach((_, i) =>
+            arm(i, waits[i] + BATCH_START_GRACE_MS, () => ({
+                status: null,
+                error: 'dispatcher: call never started',
+            })),
+        );
+        const started = (i: number): void => {
+            if (!answered[i])
+                arm(i, batch[i].call.timeoutMs, () =>
+                    failedAnswer(timeoutError()),
+                );
+        };
+        const longest = Math.max(
+            ...batch.map(
+                ({ call }, i) =>
+                    waits[i] + BATCH_START_GRACE_MS + call.timeoutMs,
+            ),
+        );
+
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_BATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                    },
+                    body,
+                    signal: AbortSignal.any([
+                        hangUp.signal,
+                        AbortSignal.timeout(longest),
+                    ]),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
+
+            const dispatchError = response.headers.get(
+                EVENTS_DISPATCH_ERROR_HEADER,
+            );
+            if (dispatchError === 'no-such-route') {
+                // A dispatcher from before the batch route: nothing ran, so
+                // every call goes again on its own.
+                void response.body?.cancel().catch(() => {});
+                this.#batchUnsupportedUntil =
+                    Date.now() + BATCH_UNSUPPORTED_RETRY_MS;
+                batch.forEach(({ call }, i) => {
+                    if (answered[i]) return;
+                    const { beforeSend } = call;
+                    if (!beforeSend) {
+                        void this.#sendOne(call).then((a) => settle(i, a));
+                        return;
+                    }
+                    // The batch round trip may have outlived the call.
+                    void (async () => {
+                        try {
+                            if (!(await beforeSend()))
+                                return settle(i, supersededAnswer());
+                        } catch (err) {
+                            return finish(i, (entry) => entry.reject(err));
+                        }
+                        settle(i, await this.#sendOne(call));
+                    })();
+                });
+                return;
+            }
+            if (dispatchError !== null || !response.ok) {
+                void response.body?.cancel().catch(() => {});
+                // An unmarked failure (an edge 404, a WAF 403) reads as
+                // Puter's, never as any handler's answer.
+                settleRest(
+                    dispatchError !== null
+                        ? dispatchAnswer(dispatchError, response.status)
+                        : {
+                              status: null,
+                              error: `dispatcher: unmarked ${response.status} for a batch`,
+                          },
+                );
+                return;
+            }
+
+            let buffered = '';
+            const read = (line: string): void => {
+                if (line.trim() === '') return;
+                const parsed = parseAnswerLine(line);
+                if (!parsed || parsed.i >= batch.length) return;
+                if ('started' in parsed) started(parsed.i);
+                else settle(parsed.i, parsed.answer);
+            };
+            const decoder = new TextDecoder();
+            for await (const chunk of response.body ?? []) {
+                buffered += decoder.decode(chunk, { stream: true });
+                let newline = buffered.indexOf('\n');
+                while (newline !== -1) {
+                    read(buffered.slice(0, newline));
+                    buffered = buffered.slice(newline + 1);
+                    newline = buffered.indexOf('\n');
+                }
+            }
+            read(buffered);
+            settleRest({
+                status: null,
+                error: 'dispatcher: batch ended without an answer',
+            });
+        } catch (err) {
+            // Every call has its answer and the request was dropped on purpose.
+            if (open === 0) return;
+            settleRest(failedAnswer(err));
+        }
+    }
+
+    /** Whether a call on `lane` may take a connection now. */
+    #free(lane: InvokeLane): boolean {
+        return (
+            this.#active < INVOKE_CONNECTIONS &&
+            (lane !== 'schedule' ||
+                this.#scheduleActive < SCHEDULE_INVOKE_CONNECTIONS)
+        );
+    }
+
+    #take(lane: InvokeLane): void {
+        this.#active++;
+        if (lane === 'schedule') this.#scheduleActive++;
+    }
+
     /** A connection slot, or false once `waitMs` passes without one. */
-    #acquire(waitMs: number): Promise<boolean> {
-        if (this.#active < INVOKE_CONNECTIONS) {
-            this.#active++;
+    #acquire(waitMs: number, lane: InvokeLane): Promise<boolean> {
+        if (this.#free(lane)) {
+            this.#take(lane);
             return Promise.resolve(true);
         }
         return new Promise((resolve) => {
@@ -420,19 +925,24 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
                 resolve(false);
             }, waitMs);
             timer.unref?.();
-            this.#waiting.add(granted);
+            this.#waiting.set(granted, lane);
         });
     }
 
-    /** Hand the slot to the longest waiter, or give it back. */
-    #release(): void {
-        const next = this.#waiting.values().next().value;
-        if (next) {
-            this.#waiting.delete(next);
-            next();
+    /**
+     * Give the slot back, then hand one to the longest waiter it can go to: a
+     * schedule waiter at a full schedule share doesn't hold up a delivery.
+     */
+    #release(lane: InvokeLane): void {
+        this.#active--;
+        if (lane === 'schedule') this.#scheduleActive--;
+        for (const [granted, waiting] of this.#waiting) {
+            if (!this.#free(waiting)) continue;
+            this.#waiting.delete(granted);
+            this.#take(waiting);
+            granted();
             return;
         }
-        this.#active--;
     }
 
     /** One pool per process, built on first use and closed with the server. */
@@ -446,6 +956,65 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
     }
 }
 
+/** What a failure the dispatcher marked as its own reads as. */
+const dispatchAnswer = (
+    reason: string,
+    status: number,
+): EventsInvokeAnswer => ({
+    status: null,
+    error: `dispatcher: ${reason} (${status})`,
+    dispatchReason: reason,
+});
+
+/** What `AbortSignal.timeout` rejects with, for a batched call's own timeout. */
+const timeoutError = (): Error =>
+    new DOMException(
+        'The operation was aborted due to timeout',
+        'TimeoutError',
+    );
+
+/** One line of a batch answer, or null when it is not one. */
+export const parseAnswerLine = (
+    line: string,
+):
+    | { i: number; started: true }
+    | { i: number; answer: EventsInvokeAnswer }
+    | null => {
+    let raw: unknown;
+    try {
+        raw = JSON.parse(line);
+    } catch {
+        return null;
+    }
+    if (!raw || typeof raw !== 'object') return null;
+    const { i, started, status, handled, error, retryAfter, dispatch } =
+        raw as Record<string, unknown>;
+    if (!Number.isInteger(i) || (i as number) < 0) return null;
+    if (started === true) return { i: i as number, started: true };
+    if (typeof status !== 'number') return null;
+    if (typeof dispatch === 'string')
+        return { i: i as number, answer: dispatchAnswer(dispatch, status) };
+    return {
+        i: i as number,
+        answer: answerFrom({
+            status,
+            headers: {
+                get: (name) => {
+                    if (name === EVENTS_HANDLED_HEADER)
+                        return handled === true ? '1' : null;
+                    if (name === EVENTS_ERROR_HEADER)
+                        return typeof error === 'string' ? error : null;
+                    if (name === 'retry-after')
+                        return typeof retryAfter === 'string'
+                            ? retryAfter
+                            : null;
+                    return null;
+                },
+            },
+        }),
+    };
+};
+
 /** Whether deploying the script is what fixes this answer. */
 const selfDeployable = (result: EventsInvokeAnswer): boolean =>
     result.status === null

src/backend/controllers/auth/AuthController.test.ts

@@ -5132,6 +5132,26 @@ describe('AuthController phone verification — staging & reuse', () => {
         expect(staged).toBe('+14155550123');
     });
 
+    it('makes an email-gate-last account verify its card before its phone', async () => {
+        const { user, actor } = await makeUserAndActor({
+            requires_phone_verification: 1,
+            requires_card_verification: 1,
+        });
+        await server.stores.user.updateMetadata(user.id, {
+            emailGateLast: true,
+        });
+        const prelude = stubPrelude();
+        await withClients({ prelude }, async () => {
+            await expect(
+                controller.handleSendConfirmPhone(
+                    makeReq({ phone: '+14155550123' }, { actor }),
+                    makeRes(),
+                ),
+            ).rejects.toMatchObject({ statusCode: 409 });
+        });
+        expect(prelude.createVerification).not.toHaveBeenCalled();
+    });
+
     it('forwards an abuse-extension veto as 429 with the opaque reason, and sends nothing', async () => {
         const { user, actor } = await makeUserAndActor();
         const emitAndWait = vi.fn(
@@ -5197,7 +5217,7 @@ describe('AuthController phone verification — staging & reuse', () => {
         );
     });
 
-    it('carries the client dispatch id on the sent signal', async () => {
+    it('carries the client dispatch id on the check and sent events', async () => {
         const { actor } = await makeUserAndActor();
         const emit = vi.fn();
         const emitAndWait = vi.fn(async () => {});
@@ -5218,6 +5238,11 @@ describe('AuthController phone verification — staging & reuse', () => {
             expect.objectContaining({ dispatch_id: 'disp-123' }),
             expect.anything(),
         );
+        expect(emitAndWait).toHaveBeenCalledWith(
+            'puter.phone-verification.check',
+            expect.objectContaining({ dispatch_id: 'disp-123' }),
+            expect.anything(),
+        );
     });
 
     it('does NOT emit the sent signal when the send fails upstream', async () => {

src/backend/controllers/auth/AuthController.ts

@@ -66,6 +66,7 @@ import {
     isCardFallbackEnabled,
     phoneAttemptsKey,
 } from '../../util/cardFallback.js';
+import { isEmailGateLast } from '../../util/verificationOrder.js';
 import { sessionCookieFlags } from '../../util/cookieFlags.js';
 import { cleanEmail, isStorableEmail } from '../../util/email.js';
 import { isGodmodeApp } from '../../util/godmodeApps.js';
@@ -1220,6 +1221,18 @@ export class AuthController extends PuterController {
         }
     }
 
+    /** The email-gate-last confirmation email; a failure never fails the gate. */
+    private async sendDeferredConfirmation(userId: number): Promise<void> {
+        try {
+            await this.services.signup.sendDeferredConfirmationEmail(userId);
+        } catch (e) {
+            console.warn(
+                '[verification] deferred confirmation email failed:',
+                e,
+            );
+        }
+    }
+
     private async isCardFallbackEligible(user: {
         id: number;
         requires_phone_verification?: boolean | number | null;
@@ -1265,6 +1278,12 @@ export class AuthController extends PuterController {
             throw new HttpError(403, 'Account suspended.', {
                 legacyCode: 'account_suspended',
             });
+        if (isEmailGateLast(user) && user.requires_card_verification)
+            throw new HttpError(
+                409,
+                'Card verification must be completed first.',
+                { legacyCode: 'conflict' },
+            );
         if (!this.clients.prelude?.isConfigured())
             throw await this.smsSendError(
                 503,
@@ -1335,6 +1354,7 @@ export class AuthController extends PuterController {
             user_uid: user.uuid,
             phone: parsed.e164,
             device_fingerprint: req.deviceFingerprint ?? null,
+            dispatch_id: dispatchId ?? null,
             allowed: true,
             reason: null as string | null,
             card_fallback_instead: false,
@@ -1573,6 +1593,7 @@ export class AuthController extends PuterController {
             requires_phone_verification: 0,
             phone: pendingPhone,
         });
+        await this.sendDeferredConfirmation(user.id);
 
         // Run the verified-event listeners synchronously (emitAndWait, not
         // fire-and-forget emit) so a carrier-based card-verification waiver in
@@ -1653,7 +1674,11 @@ export class AuthController extends PuterController {
             res.json({ card_verified: true });
             return;
         }
-        if (user.requires_phone_verification && !fallbackEligible)
+        if (
+            user.requires_phone_verification &&
+            !fallbackEligible &&
+            !isEmailGateLast(user)
+        )
             throw new HttpError(
                 409,
                 'Phone verification must be completed first.',
@@ -1716,6 +1741,7 @@ export class AuthController extends PuterController {
             await this.stores.user.update(user.id, {
                 requires_card_verification: 0,
             });
+            await this.sendDeferredConfirmation(user.id);
             res.json({ card_verified: true, disabled: true });
             return;
         }
@@ -1775,7 +1801,11 @@ export class AuthController extends PuterController {
             res.json({ card_verified: true });
             return;
         }
-        if (user.requires_phone_verification && !fallbackEligible)
+        if (
+            user.requires_phone_verification &&
+            !fallbackEligible &&
+            !isEmailGateLast(user)
+        )
             throw new HttpError(
                 409,
                 'Phone verification must be completed first.',
@@ -1816,6 +1846,7 @@ export class AuthController extends PuterController {
             await this.stores.user.update(user.id, {
                 requires_card_verification: 0,
             });
+            await this.sendDeferredConfirmation(user.id);
             res.json({ card_verified: true, disabled: true });
             return;
         }
@@ -1840,6 +1871,7 @@ export class AuthController extends PuterController {
             requires_card_verification: 0,
             ...(clearedPhoneGate ? { requires_phone_verification: 0 } : {}),
         });
+        await this.sendDeferredConfirmation(user.id);
 
         // Awaited like `user.phone-verified`, so whatever a listener grants on
         // verification is in place before the client refreshes. emitAndWait

src/backend/controllers/broadcast/BroadcastController.ts

@@ -19,6 +19,7 @@
 
 import type { Request, Response } from 'express';
 import { Controller, Post } from '../../core/http/decorators.js';
+import { schedulesUnavailable } from '../../core/schedules.js';
 import type { BroadcastService } from '../../services/broadcast/BroadcastService.js';
 import type { ForwardBatch } from '../../services/events/forwardQueue.js';
 import { PuterController } from '../types.js';
@@ -108,6 +109,42 @@ export class BroadcastController extends PuterController {
         );
         res.status(200).json({ ok: true, ...reply });
     }
+
+    /**
+     * Schedule changes a region made for schedules this one owns, handed to
+     * `services.schedules`; `events_disabled` on a server without it.
+     */
+    @Post('/events/schedules', { subdomain: '*' })
+    async schedules(req: Request, res: Response): Promise<void> {
+        const broadcast = this.services.broadcast as unknown as
+            BroadcastService | undefined;
+        if (!broadcast) {
+            res.status(503).json({
+                error: { message: 'Broadcast service not registered' },
+            });
+            return;
+        }
+
+        const verified = await broadcast.verifySignedRequest(
+            req.rawBody,
+            broadcastHeaders(req),
+        );
+        if (!verified.ok) {
+            res.status(verified.status ?? 400).json({
+                error: { message: verified.message ?? 'Bad request' },
+            });
+            return;
+        }
+        if (verified.info?.ignored) {
+            res.status(200).json({ ok: true, ...verified.info });
+            return;
+        }
+
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        const reply = await schedules.receivePeerChanges(req.body);
+        res.status(200).json({ ok: true, ...reply });
+    }
 }
 
 const broadcastHeaders = (req: Request) => {

src/backend/controllers/drivers/DriverController.test.ts

@@ -736,6 +736,42 @@ describe('DriverController driver-method lifecycle events', () => {
             rejectReason: 'blocked in test',
         });
     });
+
+    it('answers 429 when a before listener vetoes with rejectStatus 429', async () => {
+        const actor = await makeUserActor();
+        await grantKvPerm(actor);
+        const key = `throttle-${uuidv4()}`;
+        server.clients.event.on(
+            'driver.puter-kvstore.set.before',
+            (_k, data) => {
+                if ((data.args as { key?: string })?.key === key) {
+                    data.allow = false;
+                    data.rejectStatus = 429;
+                }
+            },
+        );
+
+        const req = makeReq(
+            {
+                interface: 'puter-kvstore',
+                method: 'set',
+                args: { key, value: 'v' },
+            },
+            actor,
+        );
+        await expect(
+            runWithContext({ actor }, () =>
+                routes['POST /call'](
+                    req,
+                    makeRes() as unknown as Response,
+                    () => {},
+                ),
+            ),
+        ).rejects.toMatchObject({
+            statusCode: 429,
+            legacyCode: 'too_many_requests',
+        });
+    });
 });
 
 describe('DriverController.#handleListInterfaces', () => {

src/backend/controllers/drivers/DriverController.ts

@@ -31,6 +31,7 @@ import {
     checkDriverRateLimit,
 } from '../../core/http/middleware/rateLimit.js';
 import type { PuterRouter } from '../../core/http/PuterRouter.js';
+import type { LifecycleRejectStatus } from '../../clients/event/types.js';
 import type { DriverMeta } from '../../drivers/meta.js';
 import {
     isCreditExhaustion,
@@ -334,7 +335,8 @@ export class DriverController extends PuterController {
 
         if (req.actor) {
             const permService = this.services.permission as unknown as
-                PermissionService | undefined;
+                | PermissionService
+                | undefined;
             if (permService) {
                 // Build via PermissionUtil.join so any `:` in a driver or
                 // interface name is escaped — raw interpolation would let a
@@ -475,6 +477,7 @@ export class DriverController extends PuterController {
             args,
             allow: true as boolean,
             rejectReason: undefined as string | undefined,
+            rejectStatus: undefined as LifecycleRejectStatus | undefined,
         };
         await this.clients.event?.emitAndWait(
             `driver.${ifaceName}.${method}.before`,
@@ -496,11 +499,12 @@ export class DriverController extends PuterController {
                 },
                 {},
             );
+            const throttled = beforeEvent.rejectStatus === 429;
             throw new HttpError(
-                403,
+                throttled ? 429 : 403,
                 beforeEvent.rejectReason ??
                     `Blocked by policy: ${ifaceName}:${method}`,
-                { legacyCode: 'forbidden' },
+                { legacyCode: throttled ? 'too_many_requests' : 'forbidden' },
             );
         }
 

src/backend/controllers/events/EventsController.ts

@@ -25,6 +25,11 @@ import { bodyRecord } from '../../core/http/requestBody.js';
 import { DURABLE_LIST_LIMIT_CAP } from '../../stores/events/DurableSubscriptionStore.js';
 import { EVENTS_WORKERS_LIST_LIMIT_CAP } from '../../stores/events/EventHandlerStore.js';
 import { KV_HANDLE_LIST_LIMIT_CAP } from '../../stores/events/KvShareHandleStore.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    schedulesUnavailable,
+    type ScheduleServiceContract,
+} from '../../core/schedules.js';
 import { normalizeLimit } from '../../util/pagination.js';
 import { PuterController } from '../types.js';
 import { EVENTS_WORKER_PREFIX } from '../../services/events/workerRuntime.js';
@@ -59,6 +64,8 @@ const EVENTS_SCRIPT_REGEX = new RegExp(`^${EVENTS_WORKER_PREFIX}[a-f0-9]{32}$`);
 
 /** Matches the edge dispatcher's own validation of the app uid it forwards. */
 const APP_UID_REGEX = /^app-[A-Za-z0-9_-]{1,64}$/;
+/** A user uuid. */
+const HOLDER_REGEX = /^[A-Za-z0-9-]{1,64}$/;
 
 @Controller('/events')
 export class EventsController extends PuterController {
@@ -127,7 +134,8 @@ export class EventsController extends PuterController {
      *
      * Stateless: no subscription, no stored position, nothing written. The
      * cursor comes back to the caller and goes out again as `cursor` (or
-     * `after`, the name it first shipped under).
+     * `after`, the name it first shipped under). `from=end` with no cursor
+     * starts at the newest row instead of the oldest.
      */
     @Get('/fetch', {
         subdomain: 'api',
@@ -138,9 +146,16 @@ export class EventsController extends PuterController {
     async fetch(req: Request, res: Response): Promise<void> {
         const actor = this.#requireActor(req);
         const query = (req.query ?? {}) as Record<string, unknown>;
+        const from = query.from as 'start' | 'end' | undefined;
+        if (from !== undefined && from !== 'start' && from !== 'end') {
+            throw new HttpError(400, '`from` must be `start` or `end`', {
+                legacyCode: 'bad_request',
+            });
+        }
 
         const page = await this.services.events.fetchMissed(actor, {
             subject: typeof query.subject === 'string' ? query.subject : '',
+            from,
             cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
             after: typeof query.after === 'string' ? query.after : undefined,
             limit: normalizeLimit(query.limit, {
@@ -169,6 +184,99 @@ export class EventsController extends PuterController {
         res.json({});
     }
 
+    // -- Schedules ---------------------------------------------------
+    //
+    // An app's handler run on a timer, served by `services.schedules` when a
+    // server has it and answered `events_disabled` when it doesn't. Creating
+    // one needs an app context; listing, reading and removing are scoped the
+    // way the durable verbs are. The create and remove verbs spend the
+    // subscribe and unsubscribe budgets in the service, shared with those
+    // verbs.
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /** POST /events/schedules — create a schedule for the calling app. */
+    @Post('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async createSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().createFor(actor, bodyRecord(req)));
+    }
+
+    /** GET /events/schedules — the caller's schedules, one page at a time. */
+    @Get('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async listSchedules(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        const query = (req.query ?? {}) as Record<string, unknown>;
+
+        const page = await this.#schedules().list(actor, {
+            limit: normalizeLimit(query.limit, {
+                cap: SCHEDULE_LIST_LIMIT_CAP,
+            }),
+            cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
+            includeTotal: query.includeTotal === 'true',
+        });
+
+        res.json({
+            items: page.items,
+            ...(page.cursor ? { cursor: page.cursor } : {}),
+            ...(page.total !== undefined ? { total: page.total } : {}),
+        });
+    }
+
+    @Post('/schedules/update', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async updateSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        const { scheduleId, changes } = bodyRecord(req);
+        res.json(await this.#schedules().update(actor, scheduleId, changes));
+    }
+
+    /**
+     * POST /events/schedules/remove — end one schedule. An id the caller does
+     * not hold reads as absent.
+     */
+    @Post('/schedules/remove', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async removeSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        await this.#schedules().remove(actor, bodyRecord(req).scheduleId);
+        res.json({});
+    }
+
+    /**
+     * GET /events/schedules/:scheduleId — one schedule, ended or not. An id the
+     * caller does not hold reads as absent.
+     */
+    @Get('/schedules/:scheduleId', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async getSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().get(actor, req.params.scheduleId));
+    }
+
     // -- Cross-user key-value handles --------------------------------
 
     /**
@@ -315,6 +423,7 @@ export class EventsController extends PuterController {
             handlers: await this.services.events.listHandlers(actor, {
                 appUid:
                     typeof query.appUid === 'string' ? query.appUid : undefined,
+                scope: query.scope,
             }),
         });
     }
@@ -413,6 +522,15 @@ export class EventsController extends PuterController {
             throw new HttpError(400, 'Missing or invalid `script`/`appUid`', {
                 legacyCode: 'bad_request',
             });
+        // Absent or empty is the app's own scope.
+        const holder = body.holder === undefined ? '' : body.holder;
+        if (
+            typeof holder !== 'string' ||
+            !(holder === '' || HOLDER_REGEX.test(holder))
+        )
+            throw new HttpError(400, 'Invalid `holder`', {
+                legacyCode: 'bad_request',
+            });
 
         const deployer = this.#eventsWorkerDeployer();
         if (!deployer.enabled) {
@@ -420,15 +538,25 @@ export class EventsController extends PuterController {
             return;
         }
 
-        const outcome = await deployer.ensure(appUid, script);
+        const scope = await deployer.scopeFor(appUid, holder);
+        if (!scope) {
+            res.status(404).json({ deployed: false, reason: 'no-holder' });
+            return;
+        }
+
+        const outcome = await deployer.ensure(scope, script);
         if (outcome === 'deployed') {
             res.json({ deployed: true });
             return;
         }
-        // `failed` and `throttled` are worth another attempt: the rest say
-        // this script is not what the app's handlers currently are.
+        // `failed`, `throttled` and `busy` are worth another attempt: the rest
+        // say this script is not what the app's handlers currently are.
         res.status(
-            outcome === 'failed' || outcome === 'throttled' ? 502 : 404,
+            outcome === 'failed' ||
+                outcome === 'throttled' ||
+                outcome === 'busy'
+                ? 502
+                : 404,
         ).json({
             deployed: false,
             reason: outcome,
@@ -449,14 +577,22 @@ export class EventsController extends PuterController {
     override onServerStart(): void {
         const deployer = this.#eventsWorkerDeployer();
         if (!deployer.enabled) return;
+        this.services.events.useEagerDeploy((scope, script) =>
+            deployer.deployEagerly(scope, script),
+        );
         if (this.config.workers?.localServer) {
             this.services.events.useWorkerTransport(
                 new LocalEventsInvokeTransport(this.services, deployer),
             );
             return;
         }
-        this.services.events.useWorkerMissHandler((appUid, script) =>
-            deployer.ensure(appUid, script),
+        this.services.events.useWorkerMissHandler(
+            async (appUid, script, options) => {
+                const scope = await deployer.scopeFor(appUid, options.holder);
+                return scope
+                    ? deployer.ensure(scope, script, options)
+                    : 'no-holder';
+            },
         );
     }
 

src/backend/controllers/events/limits.test.ts

@@ -34,23 +34,46 @@ import {
     EVENTS_DURABLE_SUBSCRIPTIONS_MAX,
     EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP,
     EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
+    EVENTS_HANDLER_PUBLISH_LIMIT,
+    EVENTS_HANDLER_SOURCE_MAX_BYTES,
+    EVENTS_HANDLERS_PER_APP,
     EVENTS_KV_HANDLES_PER_APP,
     EVENTS_KV_HANDLES_PER_USER,
     EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT,
     EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT,
     EVENTS_SINGLE_DELIVERY_LIMIT,
+    EVENTS_USER_HANDLERS_PER_SCOPE,
+    EVENTS_USER_SCOPE_SOURCE_MAX_BYTES,
+    EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
+    EVENTS_WORKER_DEPLOYS_PER_HOUR,
     EVENTS_WORKER_INVOCATION_LIMIT,
+    EVENTS_WORKER_SOURCE_MAX_BYTES,
+    RENTED_UNITS_PER_USER,
     limitFor,
+    tiered,
     type TieredLimit,
 } from './limits.js';
 
 const tiers: Array<[string, TieredLimit]> = [
-    ['durable subscriptions per account', EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER],
+    [
+        'durable subscriptions per account',
+        EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
+    ],
     ['durable subscriptions per app', EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP],
     ['live share handles per account', EVENTS_KV_HANDLES_PER_USER],
     ['live share handles per app', EVENTS_KV_HANDLES_PER_APP],
 ];
 
+describe('tiered', () => {
+    it('gives paid plans the base, and the free and temporary tiers their own', () => {
+        const limit = tiered(30, 20, 10);
+        expect(limitFor(limit, null)).toBe(30);
+        expect(limitFor(limit, 'some_plan')).toBe(30);
+        expect(limitFor(limit, DEFAULT_FREE_SUBSCRIPTION)).toBe(20);
+        expect(limitFor(limit, DEFAULT_TEMP_SUBSCRIPTION)).toBe(10);
+    });
+});
+
 describe('the tiered subscription quotas', () => {
     it.each(tiers)('%s never lets a free tier exceed paid', (_name, tier) => {
         for (const n of Object.values(tier.bySubscription))
@@ -105,9 +128,9 @@ describe('the tiered subscription quotas', () => {
     });
 
     it('holds an unrecognised plan to the paid base', () => {
-        expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan')).toBe(
-            EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
-        );
+        expect(
+            limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan'),
+        ).toBe(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit);
         expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, null)).toBe(
             EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
         );
@@ -131,9 +154,9 @@ describe('the delivery budgets', () => {
             expect(
                 limitFor(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT, plan),
             ).toBe(matched);
-            expect(
-                limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan),
-            ).toBe(evaluated);
+            expect(limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan)).toBe(
+                evaluated,
+            );
             expect(evaluated).toBeGreaterThanOrEqual(matched);
         }
     });
@@ -169,3 +192,42 @@ describe('the delivery budgets', () => {
         }
     });
 });
+
+describe('the handler limits', () => {
+    it('pins the app-scope numbers the quotas page lists', () => {
+        expect(EVENTS_HANDLERS_PER_APP).toBe(100);
+        expect(EVENTS_HANDLER_SOURCE_MAX_BYTES).toBe(64 * 1024);
+        expect(EVENTS_WORKER_SOURCE_MAX_BYTES).toBe(5 * 1024 * 1024);
+        expect(EVENTS_WORKER_DEPLOYS_PER_HOUR).toBe(30);
+        expect(EVENTS_HANDLER_PUBLISH_LIMIT.limit).toBe(60);
+        expect(EVENTS_HANDLER_PUBLISH_LIMIT.window).toBe(60_000);
+    });
+
+    it('pins the user-scope numbers the quotas page lists', () => {
+        expect(EVENTS_USER_HANDLERS_PER_SCOPE).toBe(20);
+        expect(EVENTS_USER_SCOPE_SOURCE_MAX_BYTES).toBe(1024 * 1024);
+        expect(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR).toBe(10);
+    });
+
+    it('keeps a user scope tighter than an app scope', () => {
+        expect(EVENTS_USER_HANDLERS_PER_SCOPE).toBeLessThan(
+            EVENTS_HANDLERS_PER_APP,
+        );
+        expect(EVENTS_USER_SCOPE_SOURCE_MAX_BYTES).toBeLessThan(
+            EVENTS_WORKER_SOURCE_MAX_BYTES,
+        );
+        expect(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR).toBeLessThan(
+            EVENTS_WORKER_DEPLOYS_PER_HOUR,
+        );
+    });
+
+    it('pins the rented-unit cap by plan', () => {
+        expect(RENTED_UNITS_PER_USER.limit).toBe(100);
+        expect(limitFor(RENTED_UNITS_PER_USER, DEFAULT_FREE_SUBSCRIPTION)).toBe(
+            16,
+        );
+        expect(limitFor(RENTED_UNITS_PER_USER, DEFAULT_TEMP_SUBSCRIPTION)).toBe(
+            16,
+        );
+    });
+});

src/backend/controllers/events/limits.ts

@@ -53,7 +53,12 @@ export interface TieredLimit {
     bySubscription: Record<string, number>;
 }
 
-const tiered = (paid: number, free: number, temp: number): TieredLimit => ({
+/** Paid, free and temporary caps as a {@link TieredLimit}. Called by extensions. */
+export const tiered = (
+    paid: number,
+    free: number,
+    temp: number,
+): TieredLimit => ({
     limit: paid,
     bySubscription: {
         [DEFAULT_FREE_SUBSCRIPTION]: free,
@@ -264,6 +269,12 @@ export const KV_SHARE_NOTIFY_LIMITS = [
  */
 export const EVENTS_HANDLERS_PER_APP = 100;
 
+/**
+ * Handlers one user may hold in their own scope of one app. Each is a script of
+ * its own, so the cap is tighter than the app's.
+ */
+export const EVENTS_USER_HANDLERS_PER_SCOPE = 20;
+
 /** Longest a serialized handler may be. */
 export const EVENTS_HANDLER_SOURCE_MAX_BYTES = 64 * 1024;
 
@@ -275,6 +286,9 @@ export const EVENTS_HANDLER_SOURCE_MAX_BYTES = 64 * 1024;
  */
 export const EVENTS_WORKER_SOURCE_MAX_BYTES = 5 * 1024 * 1024;
 
+/** Source one user scope may hold, all its handlers combined. */
+export const EVENTS_USER_SCOPE_SOURCE_MAX_BYTES = 1024 * 1024;
+
 /**
  * Deploys one app's events worker may trigger per hour.
  *
@@ -284,6 +298,16 @@ export const EVENTS_WORKER_SOURCE_MAX_BYTES = 5 * 1024 * 1024;
  */
 export const EVENTS_WORKER_DEPLOYS_PER_HOUR = 30;
 
+/** Deploys one user scope may trigger per hour. */
+export const EVENTS_USER_WORKER_DEPLOYS_PER_HOUR = 10;
+
+/**
+ * Units one user may rent at once: ordinary workers, user scopes, and the
+ * events workers of apps they own. Only creating a unit is refused; holdings
+ * over the cap are kept.
+ */
+export const RENTED_UNITS_PER_USER = tiered(100, 16, 16);
+
 /**
  * Handlers one `publishAll` may carry. A build step publishes its whole set in
  * one call, and the set is capped by what an app may hold anyway.
@@ -448,11 +472,13 @@ export const EVENTS_REGION_PENDING_CEILING = 1_000_000;
 // finish backs off the same way but is not the handler's failure, so it never
 // counts toward that run.
 
-/** Wait before the first retry of a delivery a handler could not take. */
-export const EVENTS_RETRY_BASE_MS = 2_000;
-
-/** Longest wait between retries, however many have failed. */
-export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
+// Wait before the first retry of a delivery a handler could not take, the
+// longest wait, and the doubling between them. Shared with system jobs.
+export {
+    deliveryBackoffMs,
+    EVENTS_RETRY_BASE_MS,
+    EVENTS_RETRY_MAX_MS,
+} from '../../util/retryBackoff.js';
 
 /**
  * Handler failures in a row before a subscription is suspended. Counted per
@@ -468,13 +494,6 @@ export const EVENTS_CONSECUTIVE_FAILURES = 5;
  */
 export const EVENTS_FAILURE_COUNTER_TTL_MS = 60 * 60 * 1000;
 
-/** How long to hold a delivery whose handler has failed `attempts` times. */
-export const deliveryBackoffMs = (attempts: number): number =>
-    Math.min(
-        EVENTS_RETRY_MAX_MS,
-        EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
-    );
-
 // -- Suspended backlog -----------------------------------------------
 //
 // A suspended subscription stops metering but keeps what it is owed, and that

src/backend/controllers/events/scheduleRoutes.test.ts

@@ -0,0 +1,289 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The schedule routes are thin delegates: without `services.schedules` every
+ * one answers 503 `events_disabled`; with it, each hands its request over and
+ * answers what it returns.
+ */
+
+import type { Request, Response } from 'express';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    ScheduleServiceContract,
+    ScheduleView,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { BroadcastController } from '../broadcast/BroadcastController.js';
+import type { EventsController } from './EventsController.js';
+
+let server: PuterServer;
+let events: EventsController;
+let broadcast: BroadcastController;
+let actor: Actor;
+let saved: unknown;
+
+const SCHEDULE_ID = '0b3a3c9e-9f61-4f0e-8d2a-2f1b6f8f2c11';
+
+const view = {
+    scheduleId: SCHEDULE_ID,
+    kind: 'recurring',
+    appUid: 'app-x',
+} as ScheduleView;
+
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+
+const fakeSchedules = () =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(async () => view),
+        list: vi.fn(async () => ({
+            items: [view],
+            cursor: 'next',
+            total: 3,
+        })),
+        get: vi.fn(async () => view),
+        update: vi.fn(async () => view),
+        remove: vi.fn(async () => {}),
+        receivePeerChanges: vi.fn(async () => ({ applied: 2 })),
+        prepareSendAt: vi.fn(),
+        createDeferredAppMessage: vi.fn(),
+    }) satisfies ScheduleServiceContract;
+
+const req = (init: {
+    body?: unknown;
+    query?: Record<string, unknown>;
+    params?: Record<string, string>;
+}): Request =>
+    ({
+        actor,
+        body: init.body ?? {},
+        query: init.query ?? {},
+        params: init.params ?? {},
+        headers: {},
+        rawBody: Buffer.from(JSON.stringify(init.body ?? {})),
+    }) as unknown as Request;
+
+const res = () => {
+    const sent: { status: number; body: unknown } = {
+        status: 200,
+        body: undefined,
+    };
+    const response = {
+        json: vi.fn((body: unknown) => {
+            sent.body = body;
+            return response;
+        }),
+        status: vi.fn((code: number) => {
+            sent.status = code;
+            return response;
+        }),
+        setHeader: vi.fn(() => response),
+    };
+    return { response: response as unknown as Response, sent };
+};
+
+/** Each route, called as the router would. */
+const routes: Array<[string, (response: Response) => Promise<void>]> = [
+    [
+        'POST /events/schedules',
+        (r) => events.createSchedule(req({ body: { every: 60_000 } }), r),
+    ],
+    [
+        'GET /events/schedules',
+        (r) => events.listSchedules(req({ query: { limit: '5' } }), r),
+    ],
+    [
+        'GET /events/schedules/:scheduleId',
+        (r) =>
+            events.getSchedule(req({ params: { scheduleId: SCHEDULE_ID } }), r),
+    ],
+    [
+        'POST /events/schedules/update',
+        (r) => events.updateSchedule(req({ body: { scheduleId: SCHEDULE_ID, changes: { every: '2h' } } }), r),
+    ],
+    [
+        'POST /events/schedules/remove',
+        (r) =>
+            events.removeSchedule(
+                req({ body: { scheduleId: SCHEDULE_ID } }),
+                r,
+            ),
+    ],
+    [
+        'POST /broadcast/events/schedules',
+        (r) =>
+            broadcast.schedules(
+                req({ body: { namespace: 'n', changes: [] } }),
+                r,
+            ),
+    ],
+];
+
+beforeAll(async () => {
+    server = await setupTestServer();
+    events = server.controllers.events as unknown as EventsController;
+    broadcast = server.controllers.broadcast as unknown as BroadcastController;
+    actor = makeActor({
+        user: { id: 1, uuid: 'u-1', username: 'someone' },
+    } as never);
+    saved = slot().schedules;
+    vi.spyOn(
+        server.services.broadcast,
+        'verifySignedRequest',
+    ).mockResolvedValue({ ok: true, peerId: 'peer' } as never);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+});
+
+afterAll(async () => {
+    vi.restoreAllMocks();
+    await server?.shutdown();
+});
+
+describe('without services.schedules', () => {
+    it.each(routes)('%s answers 503 events_disabled', async (_name, call) => {
+        setSchedules(undefined);
+        const { response } = res();
+        let thrown: unknown;
+        try {
+            await call(response);
+        } catch (err) {
+            thrown = err;
+        }
+        expect(isHttpError(thrown)).toBe(true);
+        expect(thrown).toMatchObject({
+            statusCode: 503,
+            legacyCode: 'events_disabled',
+            // The designed answer, so it never pages.
+            noAlarm: true,
+        });
+    });
+
+    it('refuses an unsigned peer batch before saying schedules are off', async () => {
+        setSchedules(undefined);
+        vi.spyOn(
+            server.services.broadcast,
+            'verifySignedRequest',
+        ).mockResolvedValueOnce({
+            ok: false,
+            status: 401,
+            message: 'bad signature',
+        } as never);
+        const { response, sent } = res();
+        await broadcast.schedules(
+            req({ body: { namespace: 'n', changes: [] } }),
+            response,
+        );
+        expect(sent.status).toBe(401);
+    });
+});
+
+describe('with services.schedules', () => {
+    it('creates through createFor and answers its view', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.createSchedule(req({ body: { every: 60_000 } }), response);
+        expect(schedules.createFor).toHaveBeenCalledWith(actor, {
+            every: 60_000,
+        });
+        expect(sent.body).toEqual(view);
+    });
+
+    it('lists with the limit capped, the cursor and includeTotal', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.listSchedules(
+            req({
+                query: { limit: '5000', cursor: 'c', includeTotal: 'true' },
+            }),
+            response,
+        );
+        expect(schedules.list).toHaveBeenCalledWith(actor, {
+            limit: 200,
+            cursor: 'c',
+            includeTotal: true,
+        });
+        expect(sent.body).toEqual({ items: [view], cursor: 'next', total: 3 });
+    });
+
+    it('reads one through get', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.getSchedule(
+            req({ params: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.get).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual(view);
+    });
+
+    it('updates through update and answers its view', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        const changes = { every: '2h' };
+        await events.updateSchedule(req({ body: { scheduleId: SCHEDULE_ID, changes } }), response);
+        expect(schedules.update).toHaveBeenCalledWith(actor, SCHEDULE_ID, changes);
+        expect(sent.body).toEqual(view);
+    });
+
+    it('removes through remove and answers {}', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.removeSchedule(
+            req({ body: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.remove).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual({});
+    });
+
+    it('hands a signed peer batch to receivePeerChanges', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        const body = { namespace: 'n', changes: [] };
+        await broadcast.schedules(req({ body }), response);
+        expect(schedules.receivePeerChanges).toHaveBeenCalledWith(body);
+        expect(sent).toEqual({ status: 200, body: { ok: true, applied: 2 } });
+    });
+});

src/backend/controllers/events/updateSchedule.test.ts

@@ -0,0 +1,77 @@
+import type { Request, Response } from 'express';
+import { describe, expect, it } from 'vitest';
+import { makeActor } from '../../core/actor.js';
+import type {
+    ScheduleServiceContract,
+    ScheduleView,
+} from '../../core/schedules.js';
+import { EventsController } from './EventsController.js';
+
+const actor = makeActor({
+    user: { id: 1, uuid: 'user-1', username: 'user1' } as never,
+});
+const request = (body: unknown) => ({ actor, body }) as Request;
+const controller = (schedules?: Pick<ScheduleServiceContract, 'update'>) =>
+    new EventsController(
+        {} as never,
+        {} as never,
+        {} as never,
+        { schedules } as never,
+        {} as never,
+    );
+
+describe('update schedule route contract', () => {
+    it('answers events_disabled when schedules are unavailable', async () => {
+        await expect(
+            controller().updateSchedule(
+                request({ scheduleId: 'id', changes: {} }),
+                {} as Response,
+            ),
+        ).rejects.toMatchObject({
+            legacyCode: 'events_disabled',
+            statusCode: 503,
+            noAlarm: true,
+        });
+    });
+
+    it('passes the actor, id and changes through and returns the updated view', async () => {
+        const view = {
+            scheduleId: 'id',
+            kind: 'recurring',
+            state: 'suspended',
+        } as ScheduleView;
+        const changes = { cron: '0 9 * * *', tz: 'UTC' };
+        const calls: unknown[][] = [];
+        const events = controller({
+            update: async (...args) => {
+                calls.push(args);
+                return view;
+            },
+        });
+        let sent: unknown;
+        await events.updateSchedule(request({ scheduleId: 'id', changes }), {
+            json: (value: unknown) => {
+                sent = value;
+            },
+        } as Response);
+        expect(calls).toEqual([[actor, 'id', changes]]);
+        expect(sent).toBe(view);
+    });
+
+    it('preserves validation and sending errors from the implementation', async () => {
+        const error = {
+            code: 'events_schedule_sending',
+            message: 'Already sending',
+        };
+        await expect(
+            controller({
+                update: async () => {
+                    throw error;
+                },
+            }).updateSchedule(
+                request({ scheduleId: 'id', changes: {} }),
+                {} as Response,
+            ),
+        ).rejects.toBe(error);
+    });
+});

src/backend/controllers/events/workerDeploy.test.ts

@@ -25,9 +25,35 @@
 
 import MockRedis from 'ioredis-mock';
 import { describe, expect, it } from 'vitest';
-import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
-import { EventsWorkerDeployer } from './workerDeploy.js';
-import { eventsWorkerScript } from '../../services/events/workerRuntime.js';
+import type { fetch as undiciFetch } from 'undici';
+import {
+    DispatcherInvokeTransport,
+    EVENTS_DEPLOYED_HEADER,
+    EVENTS_DISPATCH_ERROR_HEADER,
+    EVENTS_HANDLED_HEADER,
+    EventsWorkerInvokerClient,
+    type EventsInvokeCall,
+} from '../../clients/events/EventsWorkerInvokerClient.js';
+import { INTERNAL_DEPLOY_TARGET } from '../../drivers/workers/WorkerDriver.js';
+import {
+    EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
+    EVENTS_WORKER_DEPLOYS_PER_HOUR,
+} from './limits.js';
+import {
+    EventsWorkerDeployer,
+    LocalEventsInvokeTransport,
+    type EventsDeployOptions,
+    type EventsDeployOutcome,
+} from './workerDeploy.js';
+import {
+    eventsWorkerScript,
+    userHandlerScript,
+} from '../../services/events/workerRuntime.js';
+import {
+    appHandlerScope,
+    hashContent,
+    type HandlerScope,
+} from '../../stores/events/EventHandlerStore.js';
 import { handlerSetHash } from '../../services/events/workerSource.js';
 import type { IConfig } from '../../types.js';
 
@@ -46,14 +72,22 @@ interface Owner {
 type Layers = ConstructorParameters<typeof EventsWorkerDeployer>[0];
 
 /** Stub layers whose only real dependency is a fresh redis mock. */
-const makeLayers = (owners: Record<string, Owner | null>) => {
+const makeLayers = (
+    owners: Record<string, Owner | null>,
+    events: Record<string, unknown> = {},
+) => {
     const redis = new MockRedis.Cluster(['redis://localhost:7001']);
     const created: string[] = [];
+    const targets: Array<Record<string, unknown>> = [];
     /** Apps whose handler sources were read, once per read. */
     const sourceReads: string[] = [];
     const layers = {
         config: {
-            events: { workerRuntime: true, internalSecret: 'sekret' },
+            events: {
+                workerRuntime: true,
+                internalSecret: 'sekret',
+                ...events,
+            },
             // `#namespace()` needs a local-worker shape to resolve without
             // throwing — this stub never reaches a real Cloudflare or
             // Miniflare deploy either way.
@@ -63,7 +97,7 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
         stores: {
             eventHandler: {
                 setForApp: async () => [{ name: 'h', sourceHash: SOURCE_HASH }],
-                allForApp: async (appUid: string) => {
+                allForApp: async ({ appUid }: { appUid: string }) => {
                     sourceReads.push(appUid);
                     return [
                         {
@@ -86,6 +120,18 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
                 },
             },
             user: {
+                getByUuid: async (uuid: string) => {
+                    const owner = Object.values(owners).find(
+                        (o) => `uuid-${o?.ownerUserId}` === uuid,
+                    );
+                    if (!owner) return null;
+                    return {
+                        id: owner.ownerUserId,
+                        uuid,
+                        username: `user-${owner.ownerUserId}`,
+                        suspended: owner.suspended ?? false,
+                    };
+                },
                 getById: async (id: number) => {
                     const owner = Object.values(owners).find(
                         (o) => o?.ownerUserId === id,
@@ -103,8 +149,12 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
         services: {},
         drivers: {
             workers: {
-                create: async (args: { workerName: string }) => {
+                create: async (args: {
+                    workerName: string;
+                    [INTERNAL_DEPLOY_TARGET]?: Record<string, unknown>;
+                }) => {
                     created.push(args.workerName);
+                    targets.push(args[INTERNAL_DEPLOY_TARGET] ?? {});
                     return { success: true, errors: [] };
                 },
             },
@@ -113,6 +163,7 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
     return {
         layers: layers as unknown as Layers,
         created,
+        targets,
         redis,
         sourceReads,
     };
@@ -131,8 +182,8 @@ describe('EventsWorkerDeployer.ensure', () => {
         const deployer = new EventsWorkerDeployer(layers);
 
         const [a, b] = await Promise.all([
-            deployer.ensure('app-a', SCRIPT),
-            deployer.ensure('app-b', SCRIPT),
+            deployer.ensure(appHandlerScope('app-a'), SCRIPT),
+            deployer.ensure(appHandlerScope('app-b'), SCRIPT),
         ]);
         expect(a).toBe('deployed');
         expect(b).toBe('no-owner');
@@ -149,7 +200,9 @@ describe('EventsWorkerDeployer.ensure', () => {
             String(EVENTS_WORKER_DEPLOYS_PER_HOUR),
         );
 
-        expect(await deployer.ensure('app-c', SCRIPT)).toBe('throttled');
+        expect(await deployer.ensure(appHandlerScope('app-c'), SCRIPT)).toBe(
+            'throttled',
+        );
         expect(sourceReads).toEqual([]);
         expect(created).toEqual([]);
     });
@@ -164,15 +217,680 @@ describe('EventsWorkerDeployer.ensure', () => {
             '',
         );
 
-        expect(await deployer.ensure('app-e', stale)).toBe('stale');
+        expect(await deployer.ensure(appHandlerScope('app-e'), stale)).toBe(
+            'stale',
+        );
         expect(sourceReads).toEqual([]);
         const hour = new Date().toISOString().slice(0, 13);
         expect(await redis.get(`ev:deploys:{app-e}:${hour}`)).toBeNull();
     });
 
+    it('passes events.workerCpuMs to the deploy target', async () => {
+        const { layers, targets } = makeLayers(
+            { 'app-f': { ownerUserId: 6 } },
+            { workerCpuMs: 50 },
+        );
+        const deployer = new EventsWorkerDeployer(layers);
+        expect(await deployer.ensure(appHandlerScope('app-f'), SCRIPT)).toBe(
+            'deployed',
+        );
+        expect(targets[0]!.cpuMs).toBe(50);
+    });
+
+    it('passes no cpuMs when events.workerCpuMs is unset', async () => {
+        const { layers, targets } = makeLayers({ 'app-g': { ownerUserId: 7 } });
+        const deployer = new EventsWorkerDeployer(layers);
+        expect(await deployer.ensure(appHandlerScope('app-g'), SCRIPT)).toBe(
+            'deployed',
+        );
+        expect(targets[0]!.cpuMs).toBeUndefined();
+    });
+
     it('deploys normally under the cap', async () => {
         const { layers } = makeLayers({ 'app-d': { ownerUserId: 4 } });
         const deployer = new EventsWorkerDeployer(layers);
-        expect(await deployer.ensure('app-d', SCRIPT)).toBe('deployed');
+        expect(await deployer.ensure(appHandlerScope('app-d'), SCRIPT)).toBe(
+            'deployed',
+        );
+    });
+});
+
+/** A server's schedules with a deploy gate that records and can refuse. */
+const withGate = (
+    layers: Layers,
+    refuse: (capped: boolean) => boolean = () => false,
+) => {
+    const gated: Array<{ script: string; capped: boolean }> = [];
+    const gate = {
+        gateDeploy: async (
+            script: string,
+            options: { capped: boolean },
+            deploy: () => Promise<EventsDeployOutcome>,
+        ): Promise<EventsDeployOutcome> => {
+            gated.push({ script, capped: options.capped });
+            return refuse(options.capped) ? 'busy' : deploy();
+        },
+    };
+    const deployer = new EventsWorkerDeployer({
+        ...layers,
+        services: { ...layers.services, schedules: gate },
+    } as unknown as Layers);
+    return { deployer, gated };
+};
+
+describe('the schedules` deploy gate', () => {
+    it('is skipped without schedules, or with schedules that have no gate', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        await expect(
+            new EventsWorkerDeployer(layers).ensure(
+                appHandlerScope('app-x'),
+                SCRIPT,
+                {
+                    lane: 'schedule',
+                },
+            ),
+        ).resolves.toBe('deployed');
+        const gateless = new EventsWorkerDeployer({
+            ...layers,
+            services: { ...layers.services, schedules: {} },
+        } as unknown as Layers);
+        await expect(
+            gateless.ensure(appHandlerScope('app-x'), SCRIPT, {
+                lane: 'schedule',
+            }),
+        ).resolves.toBe('deployed');
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('takes every deploy, capped only for a schedule run', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers);
+
+        await expect(
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT),
+        ).resolves.toBe('deployed');
+        await expect(
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
+                lane: 'schedule',
+            }),
+        ).resolves.toBe('deployed');
+        expect(gated).toEqual([
+            { script: SCRIPT, capped: false },
+            { script: SCRIPT, capped: true },
+        ]);
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('answers a refused schedule run busy, and a delivery still deploys', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer } = withGate(layers, (capped) => capped);
+
+        await expect(
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
+                lane: 'schedule',
+            }),
+        ).resolves.toBe('busy');
+        await expect(
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT),
+        ).resolves.toBe('deployed');
+        expect(created).toEqual([SCRIPT]);
+    });
+
+    it('lets deliveries that join a schedule run`s refused deploy deploy once', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const scheduled = deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
+            lane: 'schedule',
+        });
+        const deliveries = [
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT),
+            deployer.ensure(appHandlerScope('app-x'), SCRIPT),
+        ];
+        await expect(scheduled).resolves.toBe('busy');
+        await expect(Promise.all(deliveries)).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(created).toEqual([SCRIPT]);
+        expect(gated.map((g) => g.capped)).toEqual([true, false]);
+    });
+
+    it('lets a schedule run join a delivery`s deploy in flight', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const delivery = deployer.ensure(appHandlerScope('app-x'), SCRIPT);
+        const scheduled = deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
+            lane: 'schedule',
+        });
+        await expect(Promise.all([delivery, scheduled])).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(gated).toEqual([{ script: SCRIPT, capped: false }]);
+        expect(created).toEqual([SCRIPT]);
+    });
+});
+
+describe('LocalEventsInvokeTransport', () => {
+    it('renders each request as it goes out, after a cold deploy', async () => {
+        const bodies: string[] = [];
+        let resident = false;
+        const transport = new LocalEventsInvokeTransport(
+            {
+                localworkerservice: {
+                    dispatchEventsWorker: async (request: Request) => {
+                        bodies.push(await request.text());
+                        if (!resident) return null;
+                        return new Response(null, {
+                            status: 200,
+                            headers: { 'x-puter-events-handled': '1' },
+                        });
+                    },
+                },
+            } as never,
+            {
+                ensure: async () => {
+                    resident = true;
+                    return 'deployed';
+                },
+                scopeFor: async (appUid: string) => appHandlerScope(appUid),
+            } as unknown as EventsWorkerDeployer,
+        );
+        let renders = 0;
+        const call: EventsInvokeCall = {
+            script: SCRIPT,
+            appUid: 'app-x',
+            key: 'k',
+            body: '{"n":0}',
+            timeoutMs: 5000,
+            render: () => `{"n":${++renders}}`,
+        };
+        await expect(transport.send(call)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(bodies).toEqual(['{"n":1}', '{"n":2}']);
+    });
+});
+
+describe('deploy-on-miss of an evicted script', () => {
+    it('redeploys the evicted script, then resends the run with the deployed marker', async () => {
+        const { layers, created } = makeLayers({ 'app-m': { ownerUserId: 8 } });
+        // The namespace only has the script once the deploy lands: before
+        // that the dispatcher answers a miss.
+        let resident = false;
+        layers.drivers.workers.create = (async (args: {
+            workerName: string;
+        }) => {
+            created.push(args.workerName);
+            resident = true;
+            return { success: true, errors: [] };
+        }) as never;
+        const deployer = new EventsWorkerDeployer(layers);
+
+        const sent: boolean[] = [];
+        const fetchImpl = (async (_url: string, init?: RequestInit) => {
+            const headers = (init?.headers ?? {}) as Record<string, string>;
+            sent.push(headers[EVENTS_DEPLOYED_HEADER] === '1');
+            if (resident)
+                return new Response(null, {
+                    status: 200,
+                    headers: { [EVENTS_HANDLED_HEADER]: '1' },
+                });
+            return new Response(null, {
+                status: 404,
+                headers: { [EVENTS_DISPATCH_ERROR_HEADER]: 'missing' },
+            });
+        }) as unknown as typeof undiciFetch;
+
+        const client = new EventsWorkerInvokerClient({
+            events: { workerRuntime: true },
+        } as IConfig);
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl,
+            }),
+        );
+        client.setMissHandler((appUid, script, options) =>
+            deployer.ensure(appHandlerScope(appUid), script, options),
+        );
+
+        const result = await client.invoke({
+            script: SCRIPT,
+            appUid: 'app-m',
+            handler: 'h',
+            token: 't',
+            key: 'k',
+            event: {},
+            ctx: {},
+        });
+
+        expect(result).toEqual({ outcome: 'settled', status: 200 });
+        expect(created).toEqual([SCRIPT]);
+        // One miss, then the resend carrying the deployed marker.
+        expect(sent).toEqual([false, true]);
+    });
+});
+
+// The redis mock is shared across tests, so each one uses its own app and code.
+let fresh = 0;
+const nextApp = () => `app-u${++fresh}`;
+const userScope = (holder: number, appUid: string) => ({ appUid, holder });
+const sourceN = (label: string) =>
+    `async () => { await ${label}${++fresh}(); }`;
+const scriptOf = (name: string, source: string) =>
+    userHandlerScript({ name, sourceHash: hashContent(source) }, '');
+
+type Sets = Record<number, Array<{ name: string; source: string }>>;
+
+/** Layers whose handler sets are per holder (0 is the app's own). */
+const withSets = (layers: Layers, sets: Sets) => {
+    const handlersOf = (scope: { appUid: string; holder: number }) =>
+        (sets[scope.holder] ?? []).map((h) => ({
+            appUid: scope.appUid,
+            name: h.name,
+            source: h.source,
+            sourceHash: hashContent(h.source),
+            createdAt: 0,
+            updatedAt: 0,
+        }));
+    const store = layers.stores.eventHandler as unknown as Record<
+        string,
+        unknown
+    >;
+    store.setForApp = async (scope: { appUid: string; holder: number }) =>
+        handlersOf(scope).map(({ name, sourceHash }) => ({ name, sourceHash }));
+    store.allForApp = async (scope: { appUid: string; holder: number }) =>
+        handlersOf(scope);
+};
+
+const holders = (...ids: number[]) =>
+    Object.fromEntries(
+        ids.map((id) => [`holder-${id}`, { ownerUserId: id }]),
+    ) as Record<string, Owner>;
+
+describe('EventsWorkerDeployer in a user scope', () => {
+    it('deploys the one handler whose script was asked for, as the holder', async () => {
+        const app = nextApp();
+        const a = sourceN('a');
+        const b = sourceN('b');
+        const { layers, created, targets } = makeLayers(holders(10));
+        withSets(layers, {
+            10: [
+                { name: 'a', source: a },
+                { name: 'b', source: b },
+            ],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+
+        const script = scriptOf('b', b);
+        expect(await deployer.ensure(userScope(10, app), script)).toBe(
+            'deployed',
+        );
+
+        expect(created).toEqual([script]);
+        const target = targets[0]!;
+        expect(target.tags).toEqual([`events-app:${app}`, 'user:uuid-10']);
+        // One handler's script: the other one's code is not in it.
+        expect(target.source).toContain(b);
+        expect(target.source).not.toContain(a);
+    });
+
+    it('answers a script no handler of the scope matches as stale, before the throttle', async () => {
+        const app = nextApp();
+        const { layers, redis, created } = makeLayers(holders(10));
+        withSets(layers, { 10: [{ name: 'a', source: sourceN('a') }] });
+        const deployer = new EventsWorkerDeployer(layers);
+
+        expect(
+            await deployer.ensure(
+                userScope(10, app),
+                scriptOf('b', sourceN('b')),
+            ),
+        ).toBe('stale');
+        expect(created).toEqual([]);
+        const hour = new Date().toISOString().slice(0, 13);
+        expect(await redis.get(`ev:deploys:{${app}}:10:${hour}`)).toBeNull();
+    });
+
+    it('answers a scope with no handlers as no-handlers', async () => {
+        const { layers } = makeLayers(holders(10));
+        withSets(layers, {});
+        const deployer = new EventsWorkerDeployer(layers);
+        expect(
+            await deployer.ensure(
+                userScope(10, nextApp()),
+                scriptOf('a', sourceN('a')),
+            ),
+        ).toBe('no-handlers');
+    });
+
+    it('does not deploy for a holder that is suspended or gone', async () => {
+        const source = sourceN('a');
+        const { layers, created } = makeLayers({
+            'holder-10': { ownerUserId: 10, suspended: true },
+        });
+        withSets(layers, {
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('a', source);
+        const app = nextApp();
+
+        expect(await deployer.ensure(userScope(10, app), script)).toBe(
+            'no-owner',
+        );
+        expect(await deployer.ensure(userScope(11, app), script)).toBe(
+            'no-owner',
+        );
+        expect(created).toEqual([]);
+    });
+
+    it('gives each scope its own hourly cap, apart from the app scope', async () => {
+        const app = nextApp();
+        const source = sourceN('a');
+        const { layers, created } = makeLayers({
+            ...holders(10, 11),
+            [app]: { ownerUserId: 12 },
+        });
+        withSets(layers, {
+            0: [{ name: 'a', source }],
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('a', source);
+
+        for (let i = 0; i < EVENTS_USER_WORKER_DEPLOYS_PER_HOUR; i++)
+            expect(await deployer.ensure(userScope(10, app), script)).toBe(
+                'deployed',
+            );
+        expect(await deployer.ensure(userScope(10, app), script)).toBe(
+            'throttled',
+        );
+        expect(created).toHaveLength(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR);
+
+        // Another holder of the same app, and the app's own scope, still go.
+        expect(await deployer.ensure(userScope(11, app), script)).toBe(
+            'deployed',
+        );
+        expect(await deployer.ensure(appHandlerScope(app), script)).toBe(
+            'deployed',
+        );
+    });
+
+    it('collapses concurrent deploys of one script across holders into one upload', async () => {
+        const app = nextApp();
+        const source = sourceN('a');
+        const { layers, created } = makeLayers(holders(10, 11));
+        withSets(layers, {
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('a', source);
+
+        await Promise.all([
+            deployer.ensure(userScope(10, app), script),
+            deployer.ensure(userScope(10, app), script),
+            deployer.ensure(userScope(11, app), script),
+            deployer.ensure(userScope(11, nextApp()), script),
+        ]);
+        expect(created).toHaveLength(1);
+    });
+
+    it('names a script for identical code the same under any holder and app', async () => {
+        const source = sourceN('a');
+        const { layers, created } = makeLayers(holders(10, 11));
+        withSets(layers, {
+            10: [{ name: '@x', source }],
+            11: [{ name: '@x', source }],
+        });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('@x', source);
+
+        expect(await deployer.ensure(userScope(10, nextApp()), script)).toBe(
+            'deployed',
+        );
+        expect(await deployer.ensure(userScope(11, nextApp()), script)).toBe(
+            'deployed',
+        );
+        expect(created).toEqual([script, script]);
+    });
+});
+
+describe('the deployed marker', () => {
+    const setup = () => {
+        const source = sourceN('a');
+        const made = makeLayers(holders(10, 11));
+        withSets(made.layers, {
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        return {
+            ...made,
+            deployer: new EventsWorkerDeployer(made.layers),
+            script: scriptOf('a', source),
+            app: nextApp(),
+        };
+    };
+
+    it('is left by a user-scope deploy with a ttl under a day', async () => {
+        const { deployer, redis, script, app } = setup();
+        await deployer.ensure(userScope(10, app), script);
+        const ttl = await redis.ttl(`ev:deployed:${script}`);
+        expect(ttl).toBeGreaterThan(0);
+        expect(ttl).toBeLessThan(24 * 60 * 60);
+    });
+
+    it('lets an eager deploy skip the upload another holder already made', async () => {
+        const { deployer, created, script, app } = setup();
+        expect(await deployer.deployEagerly(userScope(10, app), script)).toBe(
+            'deployed',
+        );
+        expect(await deployer.deployEagerly(userScope(11, app), script)).toBe(
+            'deployed',
+        );
+        expect(created).toEqual([script]);
+    });
+
+    it('spends no throttle on a skipped eager deploy', async () => {
+        const { deployer, redis, script, app } = setup();
+        await deployer.deployEagerly(userScope(10, app), script);
+        await deployer.deployEagerly(userScope(11, app), script);
+        const hour = new Date().toISOString().slice(0, 13);
+        expect(await redis.get(`ev:deploys:{${app}}:11:${hour}`)).toBeNull();
+    });
+
+    it('never skips a deploy-on-miss', async () => {
+        const { deployer, created, script, app } = setup();
+        await deployer.deployEagerly(userScope(10, app), script);
+        expect(await deployer.ensure(userScope(11, app), script)).toBe(
+            'deployed',
+        );
+        expect(created).toEqual([script, script]);
+    });
+
+    it('is not left by an app-scope deploy', async () => {
+        const app = nextApp();
+        const source = sourceN('a');
+        const { layers, redis, created } = makeLayers({
+            [app]: { ownerUserId: 4 },
+        });
+        withSets(layers, { 0: [{ name: 'a', source }] });
+        const deployer = new EventsWorkerDeployer(layers);
+        const script = scriptOf('a', source);
+        await deployer.ensure(appHandlerScope(app), script);
+        expect(created).toEqual([script]);
+        expect(await redis.get(`ev:deployed:${script}`)).toBeNull();
+    });
+
+    it('still deploys eagerly when the marker cannot be read', async () => {
+        const { deployer, redis, created, script, app } = setup();
+        await deployer.deployEagerly(userScope(10, app), script);
+        const get = redis.get.bind(redis);
+        redis.get = (async (key: string) => {
+            if (key.startsWith('ev:deployed:')) throw new Error('down');
+            return get(key);
+        }) as never;
+        expect(await deployer.deployEagerly(userScope(11, app), script)).toBe(
+            'deployed',
+        );
+        expect(created).toEqual([script, script]);
+    });
+});
+
+describe('scopeFor', () => {
+    it('reads no holder as the app scope, and resolves a uuid to the user scope', async () => {
+        const { layers } = makeLayers(holders(10));
+        const deployer = new EventsWorkerDeployer(layers);
+        expect(await deployer.scopeFor('app-u')).toEqual({
+            appUid: 'app-u',
+            holder: 0,
+        });
+        expect(await deployer.scopeFor('app-u', '')).toEqual({
+            appUid: 'app-u',
+            holder: 0,
+        });
+        expect(await deployer.scopeFor('app-u', 'uuid-10')).toEqual({
+            appUid: 'app-u',
+            holder: 10,
+        });
+        expect(await deployer.scopeFor('app-u', 'uuid-99')).toBeNull();
+    });
+});
+
+describe('LocalEventsInvokeTransport in a user scope', () => {
+    it('deploys the holder`s script on a miss and refuses an unknown holder', async () => {
+        const source = sourceN('a');
+        const { layers, created } = makeLayers(holders(10));
+        withSets(layers, { 10: [{ name: 'a', source }] });
+        const deployer = new EventsWorkerDeployer(layers);
+        let resident = false;
+        const transport = new LocalEventsInvokeTransport(
+            {
+                localworkerservice: {
+                    dispatchEventsWorker: async () => {
+                        if (!resident) return null;
+                        return new Response(null, {
+                            status: 200,
+                            headers: { 'x-puter-events-handled': '1' },
+                        });
+                    },
+                },
+            } as never,
+            {
+                ensure: async (
+                    scope: HandlerScope,
+                    script: string,
+                    options: EventsDeployOptions,
+                ) => {
+                    const outcome = await deployer.ensure(
+                        scope,
+                        script,
+                        options,
+                    );
+                    resident = outcome === 'deployed';
+                    return outcome;
+                },
+                scopeFor: (appUid: string, holder?: string) =>
+                    deployer.scopeFor(appUid, holder),
+            } as unknown as EventsWorkerDeployer,
+        );
+        const script = scriptOf('a', source);
+        const call: EventsInvokeCall = {
+            script,
+            appUid: nextApp(),
+            holder: 'uuid-10',
+            key: 'k',
+            body: '{}',
+            timeoutMs: 5000,
+        };
+
+        await expect(transport.send(call)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(created).toEqual([script]);
+
+        resident = false;
+        await expect(
+            transport.send({ ...call, holder: 'uuid-99' }),
+        ).resolves.toMatchObject({ status: null, error: 'deploy: no-holder' });
+        expect(created).toEqual([script]);
+    });
+});
+
+describe('concurrent eager deploys of one script', () => {
+    const setup = () => {
+        const source = sourceN('a');
+        const made = makeLayers(holders(10, 11));
+        withSets(made.layers, {
+            10: [{ name: 'a', source }],
+            11: [{ name: 'a', source }],
+        });
+        return {
+            ...made,
+            script: scriptOf('a', source),
+            app: nextApp(),
+        };
+    };
+
+    it('upload once when two holders publish the same new code together', async () => {
+        const { layers, created, script, app } = setup();
+        const deployer = new EventsWorkerDeployer(layers);
+
+        const outcomes = await Promise.all([
+            deployer.deployEagerly(userScope(10, app), script),
+            deployer.deployEagerly(userScope(11, app), script),
+        ]);
+
+        expect(outcomes).toEqual(['deployed', 'deployed']);
+        expect(created).toEqual([script]);
+    });
+
+    it('skip the upload when the marker appears while they wait on the gate', async () => {
+        const { layers, redis, created, script, app } = setup();
+        // Another node's deploy of the script finishes before the gate opens.
+        const gated = new EventsWorkerDeployer({
+            ...layers,
+            services: {
+                schedules: {
+                    gateDeploy: async (
+                        _script: string,
+                        _options: { capped: boolean },
+                        deploy: () => Promise<EventsDeployOutcome>,
+                    ) => {
+                        await redis.set(`ev:deployed:${script}`, '1');
+                        return deploy();
+                    },
+                },
+            },
+        } as unknown as Layers);
+
+        expect(
+            await gated.ensure(userScope(10, app), script, { eager: true }),
+        ).toBe('deployed');
+        expect(created).toEqual([]);
+        // A miss still uploads, whatever the marker says.
+        expect(await gated.ensure(userScope(11, app), script)).toBe('deployed');
+        expect(created).toEqual([script]);
+    });
+
+    it('make a miss that joins an eager deploy deploy after it', async () => {
+        const { layers, redis, created, script, app } = setup();
+        const deployer = new EventsWorkerDeployer(layers);
+        await redis.set(`ev:deployed:${script}`, '1');
+
+        const eager = deployer.ensure(userScope(10, app), script, {
+            eager: true,
+        });
+        const miss = deployer.ensure(userScope(11, app), script);
+
+        expect(await eager).toBe('deployed');
+        expect(await miss).toBe('deployed');
+        // The eager one skipped on the marker; the miss uploaded.
+        expect(created).toEqual([script]);
     });
 });

src/backend/controllers/events/workerDeploy.ts

@@ -29,6 +29,7 @@ import {
 import type {
     EventsInvokeAnswer,
     EventsInvokeCall,
+    InvokeLane,
 } from '../../clients/events/EventsWorkerInvokerClient.js';
 import {
     EVENTS_INVOKE_PATH,
@@ -36,47 +37,89 @@ import {
     failedAnswer,
 } from '../../clients/events/EventsWorkerInvokerClient.js';
 import type { puterServices } from '../../services/index.js';
+import type { IExtensionServiceInstances } from '../../services/types.js';
 import {
     eventsInvokeKey,
     eventsWorkerScope,
     eventsWorkerScript,
+    userHandlerScript,
 } from '../../services/events/workerRuntime.js';
 import {
     generateEventsWorkerSource,
     handlerSetHash,
 } from '../../services/events/workerSource.js';
+import {
+    appHandlerScope,
+    type HandlerScope,
+} from '../../stores/events/EventHandlerStore.js';
 import type { puterStores } from '../../stores/index.js';
 import type { IConfig, LayerInstances } from '../../types.js';
-import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
+import {
+    EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
+    EVENTS_WORKER_DEPLOYS_PER_HOUR,
+} from './limits.js';
 
 /**
  * Deploys an app's events worker from its published handlers.
  *
  * Deployed the first time a delivery needs it and again on eviction — a publish
- * only writes rows. Scripts carry no worker token and no `subdomains` row: not
- * one of the owner's workers, not addressable from the internet, not listed or
- * deleted as if it were. A republished set leaves the old script behind rather
- * than deleting it; redeploying a still-live one is always safe.
+ * only writes rows (a user-scope publish also deploys eagerly). An app's scope
+ * is one script for the whole set; a user scope is one script per handler,
+ * deployed as the holder. Scripts carry no worker token and no `subdomains`
+ * row: not one of the owner's workers, not addressable from the internet, not
+ * listed or deleted as if it were. A republished set leaves the old script
+ * behind rather than deleting it; redeploying a still-live one is always safe.
  */
 
 /** Why a deploy could not happen, for the callers that answer differently. */
 export type EventsDeployOutcome =
-    'deployed' | 'stale' | 'no-handlers' | 'no-owner' | 'throttled' | 'failed';
+    | 'deployed'
+    | 'stale'
+    | 'no-handlers'
+    | 'no-owner'
+    | 'throttled'
+    | 'failed'
+    | 'busy';
+
+/** Whose call asked for the deploy: a schedule run's is held to a share. */
+export interface EventsDeployOptions {
+    lane?: InvokeLane;
+    /**
+     * A user-scope publish's deploy, which skips the upload when the script's
+     * deployed marker is already set. A deploy-on-miss is never eager.
+     */
+    eager?: boolean;
+}
 
 interface DeployLayers {
     config: IConfig;
     clients: LayerInstances<typeof puterClients>;
     stores: LayerInstances<typeof puterStores>;
-    services: LayerInstances<typeof puterServices>;
+    services: LayerInstances<typeof puterServices> &
+        Pick<IExtensionServiceInstances, 'schedules'>;
     drivers: LayerInstances<typeof puterDrivers>;
 }
 
 /**
- * `ev:deploys:{<appUid>}:<hour>` bucket TTL — comfortably past the hour it
- * counts.
+ * `ev:deploys:{<appUid>}[:<holder>]:<hour>` bucket TTL — comfortably past the
+ * hour it counts.
  */
 const DEPLOY_THROTTLE_TTL_SECONDS = 2 * 60 * 60;
 
+/** How long `ev:deployed:<script>` says a script was uploaded. Under a day. */
+const DEPLOYED_MARKER_TTL_SECONDS = 20 * 60 * 60;
+
+const deployedMarkerKey = (script: string): string => `ev:deployed:${script}`;
+
+const isUserScope = (scope: HandlerScope): boolean => scope.holder !== 0;
+
+interface InFlightDeploy {
+    /** A schedule run's, which the region's deploy slots may refuse. */
+    capped: boolean;
+    eager: boolean;
+    promise: Promise<EventsDeployOutcome>;
+}
+
 interface DeployResult {
     success?: boolean;
     errors?: unknown[];
@@ -85,7 +128,7 @@ interface DeployResult {
 export class EventsWorkerDeployer {
     readonly #layers: DeployLayers;
     /** Collapses the burst of misses one cold app produces into one deploy. */
-    readonly #inFlight = new Map<string, Promise<EventsDeployOutcome>>();
+    readonly #inFlight = new Map<string, InFlightDeploy>();
 
     constructor(layers: DeployLayers) {
         this.#layers = layers;
@@ -100,44 +143,188 @@ export class EventsWorkerDeployer {
     }
 
     /**
-     * Deploy `script`, which must be what the app's handlers currently hash to.
-     * A request for anything else is `stale` — the set moved on, and the next
-     * delivery resolves the new script rather than this one being revived.
+     * Deploy `script`, which must be what the scope's handlers currently hash
+     * to: the whole set for an app, one handler for a user scope. A request for
+     * anything else is `stale` — the handlers moved on, and the next delivery
+     * resolves the new script rather than this one being revived. Always
+     * uploads; a deploy-on-miss means whatever said it was there is wrong.
      */
-    ensure(appUid: string, script: string): Promise<EventsDeployOutcome> {
-        const key = `${appUid}|${script}`;
+    ensure(
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions = {},
+    ): Promise<EventsDeployOutcome> {
+        // A user scope's script is the same bytes whoever holds it, so
+        // concurrent deploys of it from any holder or app are one.
+        const key = isUserScope(scope)
+            ? `user|${script}`
+            : `${scope.appUid}|${scope.holder}|${script}`;
+        const capped = options.lane === 'schedule';
         const existing = this.#inFlight.get(key);
-        if (existing) return existing;
-        const started = this.#deploy(appUid, script).finally(() =>
-            this.#inFlight.delete(key),
+        if (!existing) return this.#start(key, scope, script, options);
+        // An eager deploy may have skipped the upload, which a miss must not
+        // take for done.
+        if (existing.eager && !options.eager)
+            return this.#afterEager(
+                existing.promise,
+                key,
+                scope,
+                script,
+                options,
+            );
+        if (capped || !existing.capped) return existing.promise;
+        return this.#afterCapped(existing.promise, key, scope, script, options);
+    }
+
+    /**
+     * {@link ensure} for the deploy a user-scope publish starts, which skips the
+     * upload when another holder's deploy of the same script left its marker.
+     * Only this path reads the marker.
+     */
+    async deployEagerly(
+        scope: HandlerScope,
+        script: string,
+    ): Promise<EventsDeployOutcome> {
+        if (await this.#markedDeployed(script)) return 'deployed';
+        return this.ensure(scope, script, { eager: true });
+    }
+
+    /**
+     * The scope a script request names: the app's own, or the user scope of
+     * `holder` (a user uuid). Null for a holder that does not exist.
+     */
+    async scopeFor(
+        appUid: string,
+        holder?: string | null,
+    ): Promise<HandlerScope | null> {
+        if (!holder) return appHandlerScope(appUid);
+        const user = await this.#layers.stores.user.getByUuid(holder);
+        if (!user?.id) return null;
+        return { appUid, holder: Number(user.id) };
+    }
+
+    #start(
+        key: string,
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const entry: InFlightDeploy = {
+            capped: options.lane === 'schedule',
+            eager: options.eager === true,
+            promise: this.#gated(scope, script, options).finally(() => {
+                if (this.#inFlight.get(key) === entry)
+                    this.#inFlight.delete(key);
+            }),
+        };
+        this.#inFlight.set(key, entry);
+        return entry.promise;
+    }
+
+    /** A miss that found an eager deploy in flight deploys after it. */
+    async #afterEager(
+        eager: Promise<EventsDeployOutcome>,
+        key: string,
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        await eager;
+        const current = this.#inFlight.get(key);
+        if (current && !current.eager) return current.promise;
+        return this.#start(key, scope, script, options);
+    }
+
+    /**
+     * A delivery that joined a schedule run's deploy: `busy` means no deploy
+     * started, and a delivery is never refused a slot, so it asks again.
+     */
+    async #afterCapped(
+        capped: Promise<EventsDeployOutcome>,
+        key: string,
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const outcome = await capped;
+        if (outcome !== 'busy') return outcome;
+        const current = this.#inFlight.get(key);
+        if (current && !current.capped) return current.promise;
+        return this.#start(key, scope, script, options);
+    }
+
+    /**
+     * Every deploy goes through the schedules' deploy gate when a server has
+     * one: a per-script lock, so a burst of runs of an evicted script deploys
+     * it once, and a region-wide cap that a schedule run's deploy (`capped`)
+     * may be refused with `busy`. A delivery's deploy is never refused.
+     */
+    #gated(
+        scope: HandlerScope,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const deploy = () =>
+            this.#deploy(scope, script, options.eager === true);
+        return (
+            this.#layers.services.schedules?.gateDeploy?.(
+                script,
+                { capped: options.lane === 'schedule' },
+                deploy,
+            ) ?? deploy()
         );
-        this.#inFlight.set(key, started);
-        return started;
     }
 
     async #deploy(
-        appUid: string,
+        handlerScope: HandlerScope,
         script: string,
+        eager: boolean,
     ): Promise<EventsDeployOutcome> {
+        const { appUid } = handlerScope;
         const { stores, drivers } = this.#layers;
-        const scope = eventsWorkerScope(this.#layers.config);
+        const backend = eventsWorkerScope(this.#layers.config);
+        const userScope = isUserScope(handlerScope);
 
         // Names and hashes are enough to tell a stale request, so the set's
         // sources are read and built only once the throttle has let it through.
-        const set = await stores.eventHandler.setForApp(appUid);
+        const set = await stores.eventHandler.setForApp(handlerScope);
         if (set.length === 0) return 'no-handlers';
         // Two apps that published byte-identical sets share one script, which
         // is sound: the source is the same, and everything per-app arrives with
-        // the invocation. Whoever deploys first owns it.
-        if (eventsWorkerScript(handlerSetHash(set), scope) !== script)
+        // the invocation. Whoever deploys first owns it. A user scope's script
+        // is one handler's, so it is shared the same way across holders.
+        const wanted = userScope
+            ? set.find((entry) => userHandlerScript(entry, backend) === script)
+            : undefined;
+        if (
+            userScope
+                ? !wanted
+                : eventsWorkerScript(handlerSetHash(set), backend) !== script
+        )
             return 'stale';
 
-        if (await this.#tooManyDeploysThisHour(appUid)) return 'throttled';
+        // Asked again inside the gate, where another deploy of this script may
+        // have just finished.
+        if (eager && userScope && (await this.#markedDeployed(script)))
+            return 'deployed';
 
-        const handlers = await stores.eventHandler.allForApp(appUid);
+        if (await this.#tooManyDeploysThisHour(handlerScope))
+            return 'throttled';
+
+        const all = await stores.eventHandler.allForApp(handlerScope);
+        const handlers = wanted
+            ? all.filter(
+                  (handler) =>
+                      handler.name === wanted.name &&
+                      handler.sourceHash === wanted.sourceHash,
+              )
+            : all;
         const generated = generateEventsWorkerSource(handlers);
         // The set can move between the two reads.
-        if (eventsWorkerScript(generated.setHash, scope) !== script)
+        if (
+            handlers.length === 0 ||
+            eventsWorkerScript(generated.setHash, backend) !== script
+        )
             return 'stale';
         if (generated.tooLarge) {
             console.error(
@@ -151,7 +338,7 @@ export class EventsWorkerDeployer {
                 generated.broken.join(', '),
             );
 
-        const owner = await this.#owner(appUid);
+        const owner = await this.#owner(handlerScope);
         if (!owner) return 'no-owner';
 
         const secret = this.#layers.config.events?.internalSecret;
@@ -180,6 +367,7 @@ export class EventsWorkerDeployer {
                         runtime: 'events',
                         source: generated.source,
                         omitOwnerToken: true,
+                        cpuMs: this.#layers.config.events?.workerCpuMs,
                         // What the script compares an invocation against.
                         // Derived from the script name, so a redeploy of
                         // the same set binds the same key.
@@ -210,34 +398,74 @@ export class EventsWorkerDeployer {
             );
             return 'failed';
         }
+        if (userScope) await this.#markDeployed(script);
         return 'deployed';
     }
 
-    /**
-     * Whether this app has already deployed `EVENTS_WORKER_DEPLOYS_PER_HOUR`
-     * times this hour. A flapping set (publish, remove, republish) provisions
-     * upstream on every miss, and this is the ceiling on that loop.
-     */
-    async #tooManyDeploysThisHour(appUid: string): Promise<boolean> {
-        const redis = this.#layers.clients.redis;
-        const hour = new Date().toISOString().slice(0, 13);
-        const key = `ev:deploys:{${appUid}}:${hour}`;
-        const count = await redis.incr(key);
-        if (count === 1) await redis.expire(key, DEPLOY_THROTTLE_TTL_SECONDS);
-        return count > EVENTS_WORKER_DEPLOYS_PER_HOUR;
+    async #markedDeployed(script: string): Promise<boolean> {
+        try {
+            return !!(await this.#layers.clients.redis.get(
+                deployedMarkerKey(script),
+            ));
+        } catch (err) {
+            console.warn('[events] deployed marker unreadable', err);
+            return false;
+        }
     }
 
-    /** The app owner, as an actor the deploy can run as. */
+    async #markDeployed(script: string): Promise<void> {
+        try {
+            await this.#layers.clients.redis.set(
+                deployedMarkerKey(script),
+                '1',
+                'EX',
+                DEPLOYED_MARKER_TTL_SECONDS,
+            );
+        } catch (err) {
+            console.warn('[events] could not mark a script deployed', err);
+        }
+    }
+
+    /**
+     * Whether this scope has already deployed its hourly cap this hour. A
+     * flapping set (publish, remove, republish) provisions upstream on every
+     * miss, and this is the ceiling on that loop. An app's counter is per app;
+     * a user scope has its own, so one user cannot spend another's.
+     */
+    async #tooManyDeploysThisHour(scope: HandlerScope): Promise<boolean> {
+        const redis = this.#layers.clients.redis;
+        const hour = new Date().toISOString().slice(0, 13);
+        const userScope = isUserScope(scope);
+        const key = userScope
+            ? `ev:deploys:{${scope.appUid}}:${scope.holder}:${hour}`
+            : `ev:deploys:{${scope.appUid}}:${hour}`;
+        const count = await redis.incr(key);
+        if (count === 1) await redis.expire(key, DEPLOY_THROTTLE_TTL_SECONDS);
+        return (
+            count >
+            (userScope
+                ? EVENTS_USER_WORKER_DEPLOYS_PER_HOUR
+                : EVENTS_WORKER_DEPLOYS_PER_HOUR)
+        );
+    }
+
+    /**
+     * The user a scope's deploy runs as, as an actor: the app's owner, or the
+     * holder of a user scope.
+     */
     async #owner(
-        appUid: string,
+        scope: HandlerScope,
     ): Promise<
         | (Actor & { user: { id: number; uuid: string; username: string } })
         | null
     > {
         const { stores } = this.#layers;
-        const app = await stores.app.getByUid(appUid);
-        const ownerUserId = Number(app?.owner_user_id);
-        if (!app || !Number.isFinite(ownerUserId)) return null;
+        let ownerUserId = scope.holder;
+        if (!isUserScope(scope)) {
+            const app = await stores.app.getByUid(scope.appUid);
+            ownerUserId = Number(app?.owner_user_id);
+            if (!app || !Number.isFinite(ownerUserId)) return null;
+        }
 
         const user = await stores.user.getById(ownerUserId);
         if (!user?.id || !user.uuid || !user.username || user.suspended)
@@ -299,6 +527,8 @@ export class LocalEventsInvokeTransport {
     }
 
     async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        // Built per attempt, so a send time stamped into the body is taken
+        // after a cold deploy rather than before it.
         const request = (): Request =>
             new Request(
                 `http://${call.script}.workers.puter.localhost${EVENTS_INVOKE_PATH}`,
@@ -308,7 +538,7 @@ export class LocalEventsInvokeTransport {
                         'content-type': 'application/json',
                         'x-puter-events-key': call.key,
                     },
-                    body: call.body,
+                    body: call.render ? call.render() : call.body,
                 },
             );
 
@@ -316,12 +546,20 @@ export class LocalEventsInvokeTransport {
             const resident = await this.#dispatch(request(), call.timeoutMs);
             if (resident !== null) return resident;
 
-            const outcome = await this.#deployer.ensure(
+            const scope = await this.#deployer.scopeFor(
                 call.appUid,
-                call.script,
+                call.holder,
             );
+            if (!scope) return { status: null, error: 'deploy: no-holder' };
+            const outcome = await this.#deployer.ensure(scope, call.script, {
+                lane: call.lane,
+            });
             if (outcome !== 'deployed')
-                return { status: null, error: `deploy: ${outcome}` };
+                return {
+                    status: null,
+                    error: `deploy: ${outcome}`,
+                    ...(outcome === 'busy' ? { deployBusy: true } : {}),
+                };
 
             const retried = await this.#dispatch(request(), call.timeoutMs);
             if (retried === null)

src/backend/controllers/oidc/OIDCController.test.ts

@@ -244,7 +244,10 @@ describe('OIDCController GET /auth/oidc/providers', () => {
         await callRoute('get', '/auth/oidc/providers', makeReq({}), res);
         // We configured a single `custom` provider; the live service
         // walks the config and returns its id.
-        expect(captured.body).toEqual({ providers: ['custom'] });
+        expect(captured.body).toEqual({
+            providers: ['custom'],
+            signupOidcOnly: false,
+        });
     });
 });
 

src/backend/controllers/oidc/OIDCController.ts

@@ -396,9 +396,12 @@ export class OIDCController extends PuterController {
                 },
             },
             async (_req: Request, res: Response) => {
-                const providers =
-                    await this.services.oidc.getEnabledProviderIds();
-                res.json({ providers });
+                const [providers, policy] = await Promise.all([
+                    this.services.oidc.getEnabledProviderIds(),
+                    this.services.signup.signupPolicy(),
+                ]);
+                // The signup window drops its email form when this is set.
+                res.json({ providers, signupOidcOnly: policy.oidcOnly });
             },
         );
 

src/backend/controllers/static/StaticAssetsController.test.ts

@@ -19,8 +19,9 @@
 
 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
+import type { AddressInfo } from 'node:net';
 import path from 'node:path';
-import type { Request, Response } from 'express';
+import express, { type Request, type Response } from 'express';
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { PuterRouter } from '../../core/http/PuterRouter.js';
 import type { RouteDescriptor } from '../../core/http/types.js';
@@ -111,6 +112,7 @@ const callGetHandler = async (
     let captured: { filename?: string; root?: string } = {};
     const req = {} as Request;
     const res = {
+        setHeader: vi.fn(),
         sendFile: vi.fn((filename: string, opts: { root: string }) => {
             captured = { filename, root: opts.root };
         }),
@@ -238,6 +240,48 @@ describe('StaticAssetsController puterjs_root', () => {
     });
 });
 
+// ── push-sw.js caching ──────────────────────────────────────────────
+
+/** Serves the route's handler over real HTTP and returns the response. */
+const fetchThroughExpress = async (router: PuterRouter, routePath: string) => {
+    const route = findRoute(router, 'get', routePath);
+    if (!route) throw new Error(`No GET ${routePath} registered`);
+    const app = express();
+    app.get(routePath, (req, res, next) => route.handler(req, res, next));
+    const listener = app.listen(0);
+    await new Promise((resolve) => listener.once('listening', resolve));
+    try {
+        const { port } = listener.address() as AddressInfo;
+        return await fetch(`http://127.0.0.1:${port}${routePath}`);
+    } finally {
+        listener.close();
+    }
+};
+
+describe('StaticAssetsController push-sw.js', () => {
+    it.each([
+        ['client_libs_root', 'puter.js/push-sw.js'],
+        ['puterjs_root', 'push-sw.js'],
+    ] as const)(
+        'lets the CDN keep it for five minutes from %s',
+        async (configKey, file) => {
+            const root = makeTempDir();
+            mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
+            writeFileSync(path.join(root, file), '/* worker */');
+            const router = buildController({ [configKey]: root });
+
+            for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
+                const response = await fetchThroughExpress(router, route);
+                expect(response.status).toBe(200);
+                expect(response.headers.get('cache-control')).toBe(
+                    'public, max-age=0, s-maxage=300',
+                );
+                expect(await response.text()).toBe('/* worker */');
+            }
+        },
+    );
+});
+
 // ── gui_assets_root ─────────────────────────────────────────────────
 
 describe('StaticAssetsController gui_assets_root', () => {

src/backend/controllers/static/StaticAssetsController.ts

@@ -23,6 +23,9 @@ import path from 'node:path';
 import { PuterController } from '../types.js';
 import type { PuterRouter } from '../../core/http/PuterRouter';
 
+// Sites import push-sw.js by URL, so the CDN refetches it within five minutes.
+const PUSH_WORKER_CACHE_CONTROL = 'public, max-age=0, s-maxage=300';
+
 /**
  * Static asset routes.
  *
@@ -58,6 +61,7 @@ export class StaticAssetsController extends PuterController {
             // `/v2/push-sw.js` is the path older docs gave.
             for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
                 router.get(route, { subdomain: 'js' }, (_req, res) => {
+                    res.setHeader('Cache-Control', PUSH_WORKER_CACHE_CONTROL);
                     res.sendFile('puter.js/push-sw.js', { root });
                 });
             }
@@ -108,6 +112,10 @@ export class StaticAssetsController extends PuterController {
                 });
                 for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
                     router.get(route, { subdomain: 'js' }, (_req, res) => {
+                        res.setHeader(
+                            'Cache-Control',
+                            PUSH_WORKER_CACHE_CONTROL,
+                        );
                         res.sendFile('push-sw.js', { root: puterjsRoot });
                     });
                 }

src/backend/core/http/middleware/gates.test.ts

@@ -836,6 +836,34 @@ describe('requireVerifiedAccount', () => {
         expectHttpError(got, 403, 'card_verification_required');
     });
 
+    it('asks an email-gate-last account for card, then phone, then email', () => {
+        const pending = {
+            uuid: 'u-1',
+            requires_email_confirmation: true,
+            email_confirmed: false,
+            requires_phone_verification: true,
+            requires_card_verification: true,
+            metadata: { emailGateLast: true },
+        };
+        const gateFor = (user: Record<string, unknown>) =>
+            runGate(requireVerifiedAccount(), { actor: { user } });
+        expectHttpError(gateFor(pending), 403, 'card_verification_required');
+        expectHttpError(
+            gateFor({ ...pending, requires_card_verification: false }),
+            403,
+            'phone_verification_required',
+        );
+        expectHttpError(
+            gateFor({
+                ...pending,
+                requires_card_verification: false,
+                requires_phone_verification: false,
+            }),
+            403,
+            'email_confirmation_required',
+        );
+    });
+
     it('returns 403 two_factor_required for a seat whose team requires 2FA', async () => {
         const teams = { getOrgSeat: async () => ({ require_2fa: 1 }) };
         const got = await runGateAsync(requireTeam2fa(teams), {

src/backend/core/http/middleware/gates.ts

@@ -32,6 +32,7 @@ import {
     type CardFallbackDeps,
 } from '../../../util/cardFallback';
 import { reserveUsernames } from '../../../util/reservedUsernames';
+import { isEmailGateLast } from '../../../util/verificationOrder';
 import {
     isAccountContext,
     isAppActor,
@@ -309,11 +310,25 @@ export const requireVerifiedAccount = (): RequestHandler =>
 export const assertVerifiedAccount = (
     user: AccountGateUser | undefined,
 ): void => {
-    if (user?.requires_email_confirmation && !user?.email_confirmed) {
-        throw new HttpError(403, 'Please confirm your email to continue', {
-            legacyCode: 'email_confirmation_required',
-        });
-    }
+    const assertEmail = () => {
+        if (user?.requires_email_confirmation && !user?.email_confirmed) {
+            throw new HttpError(403, 'Please confirm your email to continue', {
+                legacyCode: 'email_confirmation_required',
+            });
+        }
+    };
+    const assertCard = () => {
+        if (user?.requires_card_verification) {
+            throw new HttpError(403, 'Please verify your card to continue', {
+                legacyCode: 'card_verification_required',
+            });
+        }
+    };
+    // Email → phone → card, or card → phone → email for an email-gate-last
+    // account, so the client is sent to the gate it has to clear next.
+    const emailLast = isEmailGateLast(user);
+    if (emailLast) assertCard();
+    else assertEmail();
     if (user?.requires_phone_verification) {
         throw new HttpError(
             403,
@@ -323,11 +338,8 @@ export const assertVerifiedAccount = (
             },
         );
     }
-    if (user?.requires_card_verification) {
-        throw new HttpError(403, 'Please verify your card to continue', {
-            legacyCode: 'card_verification_required',
-        });
-    }
+    if (emailLast) assertEmail();
+    else assertCard();
     // A team seat signs in on a password its administrator still holds,
     // so it reaches nothing until it has replaced that password.
     if (user?.requires_password_change) {

src/backend/core/http/routeLifecycle.test.ts

@@ -290,6 +290,26 @@ describe('createRouteLifecycleMiddleware', () => {
         expect(after).toHaveLength(0);
     });
 
+    it('answers 429 when the vetoing listener asks for it', async () => {
+        const events = makeEvents();
+        events.on(`${BASE}.before` as never, (_k, data) => {
+            (data as RouteLifecycleEvent).allow = false;
+            (data as RouteLifecycleEvent).rejectReason = 'slow down';
+            (data as RouteLifecycleEvent).rejectStatus = 429;
+        });
+        const reject = record(events, `${BASE}.reject` as keyof EventMap);
+
+        const mw = createRouteLifecycleMiddleware(events, POST, PATH);
+        const res = new FakeRes();
+        await mw(makeReq('u-9'), res as unknown as Response, () => {});
+
+        expect(res.statusCode).toBe(429);
+        expect(res.body).toMatchObject({
+            error: { code: 'too_many_requests', message: 'slow down' },
+        });
+        expect(reject[0]).toMatchObject({ statusCode: 429 });
+    });
+
     it('treats a listener-sent response as a terminal after (not a reject) with the real status', async () => {
         const events = makeEvents();
         events.on(`${BASE}.before` as never, (_k, data) => {

src/backend/core/http/routeLifecycle.ts

@@ -20,6 +20,7 @@
 import type { Request, RequestHandler } from 'express';
 import { actorUid } from '../actor';
 import type { EventClient } from '../../clients/event/EventClient';
+import type { LifecycleRejectStatus } from '../../clients/event/types';
 import type { RouteMethod } from './types';
 
 type RoutePathValue = string | RegExp | Array<string | RegExp> | undefined;
@@ -100,30 +101,33 @@ export const createRouteLifecycleMiddleware = (
             ...base,
             allow: true as boolean,
             rejectReason: undefined as string | undefined,
+            rejectStatus: undefined as LifecycleRejectStatus | undefined,
         };
         if (observeBefore) {
             await events.emitAndWait(beforeKey, beforeEvent, {});
         }
 
         // Explicit veto: a listener set `allow = false`. Emit `reject` and
-        // answer 403 — unless the listener already wrote its own response, in
-        // which case we report that status rather than a misleading 403.
+        // answer 403 (or the listener's 429) — unless the listener already
+        // wrote its own response, in which case we report that status.
         if (beforeEvent.allow === false) {
+            const status = beforeEvent.rejectStatus === 429 ? 429 : 403;
             events.emit(
                 `${keyBase}.reject`,
                 {
                     phase: 'reject',
                     ...base,
-                    statusCode: res.headersSent ? res.statusCode : 403,
+                    statusCode: res.headersSent ? res.statusCode : status,
                     durationMs: Date.now() - startedAt,
                     rejectReason: beforeEvent.rejectReason,
                 },
                 {},
             );
             if (!res.headersSent) {
-                res.status(403).json({
+                res.status(status).json({
                     error: {
-                        code: 'forbidden',
+                        code:
+                            status === 429 ? 'too_many_requests' : 'forbidden',
                         message:
                             beforeEvent.rejectReason ?? 'Blocked by policy',
                     },

src/backend/core/http/types.ts

@@ -76,6 +76,7 @@ export type AccountGateUser = Partial<
         | 'phone'
         | 'card_fingerprint'
         | 'otp_enabled'
+        | 'metadata'
     >
 > & { id?: number };
 

src/backend/core/schedules.test.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { describe, expect, it } from 'vitest';
+import { isHttpError } from './http/HttpError.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    SCHEDULE_PEER_PATH,
+    schedulesUnavailable,
+} from './schedules.js';
+
+describe('schedules contract', () => {
+    it('answers an absent schedules service with a 503 events_disabled', () => {
+        const err = schedulesUnavailable();
+        expect(isHttpError(err)).toBe(true);
+        expect(err.statusCode).toBe(503);
+        expect(err.legacyCode).toBe('events_disabled');
+        // Expected on a server without schedules, so it must not page.
+        expect(err.noAlarm).toBe(true);
+    });
+
+    it('keeps the list cap and the peer path the extension relies on', () => {
+        expect(SCHEDULE_LIST_LIMIT_CAP).toBe(200);
+        expect(SCHEDULE_PEER_PATH).toBe('/broadcast/events/schedules');
+    });
+});

src/backend/core/schedules.ts

@@ -0,0 +1,198 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * What core asks of schedules. Nothing here is an implementation: an extension
+ * registers `services.schedules`, and a server without one answers every
+ * schedule route and `notify({ sendAt })` with `events_disabled`. Called by
+ * extensions; keep these signatures stable.
+ */
+
+import type { EventsDeployOutcome } from '../controllers/events/workerDeploy.js';
+import type { AppMessage } from '../services/notification/appMessage.js';
+import type { SuspendedReason } from '../stores/events/DurableSubscriptionStore.js';
+import type { DurableSubscription } from '../stores/events/types.js';
+import type { PageResult } from '../util/pagination.js';
+import type { Actor } from './actor.js';
+import { HttpError } from './http/HttpError.js';
+
+/** Why a schedule ended. */
+export type ScheduleEndReason =
+    | 'removed'
+    | 'ended'
+    | 'app_deleted'
+    | 'uninstalled'
+    | 'permission_revoked'
+    | 'holder_gone'
+    | 'app_gone'
+    | 'orphaned'
+    | 'delivered'
+    | 'dropped';
+
+/** What happened to a set of subscription rows schedules own. */
+export type ScheduleSubscriptionChange =
+    | {
+          type: 'suspended';
+          reason: SuspendedReason;
+          /** For a reason that never lifts; defaults to that reason. */
+          endReason?: ScheduleEndReason;
+      }
+    | { type: 'resumed' }
+    | { type: 'ended'; endReason: ScheduleEndReason };
+
+/** A schedule that keeps running, one that runs once, or a delayed notification. */
+export type ScheduleViewKind = 'recurring' | 'once' | 'deferred';
+
+/**
+ * A schedule as its holder sees it. Times are epoch ms. `context` values are
+ * never returned, as for a persistent subscription: only its keys and a hash.
+ */
+export interface ScheduleView {
+    scheduleId: string;
+    kind: ScheduleViewKind;
+    appUid: string;
+    /** `null` once ended: the handler binding goes with the schedule. */
+    handlerName: string | null;
+    /** Set only when the handler lives in the holder's own scope. */
+    handlerScope?: 'user';
+    contextKeys: string[] | null;
+    contextHash: string | null;
+    /** Interval of an `every` schedule. */
+    every: number | null;
+    cron: string | null;
+    tz: string;
+    /** First allowed run; a one-shot's only run. */
+    startAt: number;
+    /** Last allowed run, inclusive. */
+    endsAt: number | null;
+    /** `sending` once a delayed send has started; it can't be cancelled then. */
+    state: 'active' | 'suspended' | 'sending' | 'ended';
+    suspendedReason: string | null;
+    endReason: string | null;
+    endedAt: number | null;
+    /**
+     * The slot it runs for next; `null` while suspended or sending, and once
+     * ended.
+     */
+    nextRunAt: number | null;
+    createdAt: number;
+}
+
+export interface ScheduleListRequest {
+    limit: number;
+    cursor?: string;
+    includeTotal: boolean;
+}
+
+/** The caps a holder's schedules are held to, from their plan. */
+export interface ScheduleQuota {
+    recurringPerUser: number;
+    recurringPerApp: number;
+    oneShotsPerUser: number;
+    oneShotsPerApp: number;
+    /** Worst-case runs per hour × 1000, summed over recurring rows. */
+    budgetPerUser: number;
+    budgetPerApp: number;
+}
+
+/** A caller cleared to supply `sendAt`, and when it asked for. */
+export interface DeferredSendPlan {
+    sendAt: number;
+    /** Due within 5 seconds, or past: sent now rather than stored. */
+    immediate: boolean;
+    holderUserId: number;
+    appUid: string;
+    handlerDepth: number;
+    quota: ScheduleQuota;
+}
+
+/** What a delayed app message stores until it is sent. */
+export interface DeferredAppMessage {
+    message: AppMessage;
+    /** A KV share handle naming the recipient. */
+    to?: string;
+}
+
+/**
+ * A condition on one row of another table that statements embed as `EXISTS
+ * (SELECT 1 FROM <table> WHERE <where>)`, binding `values`.
+ */
+export interface SqlGuard {
+    table: string;
+    where: string;
+    values: unknown[];
+}
+
+/** Ceiling of a schedule listing page. */
+export const SCHEDULE_LIST_LIMIT_CAP = 200;
+
+/** The signed peer route a region's schedule changes are sent to. */
+export const SCHEDULE_PEER_PATH = '/broadcast/events/schedules';
+
+export const schedulesUnavailable = (): HttpError =>
+    new HttpError(503, 'Schedules are not available on this server', {
+        legacyCode: 'events_disabled',
+        // The designed answer of a server without schedules, not a fault.
+        noAlarm: true,
+    });
+
+export interface ScheduleServiceContract {
+    /**
+     * Keep schedules in step with a change to the subscription rows they own.
+     * Never throws: the subscription change has already happened.
+     */
+    onSubscriptionsChanged(
+        rows: readonly DurableSubscription[],
+        change: ScheduleSubscriptionChange,
+    ): Promise<void>;
+    createFor(
+        actor: Actor,
+        body: Record<string, unknown>,
+    ): Promise<ScheduleView>;
+    list(
+        actor: Actor,
+        request: ScheduleListRequest,
+    ): Promise<PageResult<ScheduleView>>;
+    get(actor: Actor, scheduleId: unknown): Promise<ScheduleView>;
+    update(
+        actor: Actor,
+        scheduleId: unknown,
+        changes: unknown,
+    ): Promise<ScheduleView>;
+    remove(actor: Actor, scheduleId: unknown): Promise<void>;
+    /** `results`: each change's index outcome, in order (`null` if dropped). */
+    receivePeerChanges(
+        body: unknown,
+    ): Promise<{ applied: number; results?: Array<number | null> }>;
+    /** Checks a caller supplying `sendAt`; `events_disabled` when off. */
+    prepareSendAt(actor: Actor, sendAt: unknown): Promise<DeferredSendPlan>;
+    createDeferredAppMessage(
+        plan: DeferredSendPlan,
+        message: DeferredAppMessage,
+    ): Promise<{ scheduleId: string; sendAt: number }>;
+    /**
+     * Runs an events worker deploy under the schedules' deploy gate; `capped`
+     * for a schedule run's. A passthrough when schedules are off.
+     */
+    gateDeploy?(
+        script: string,
+        options: { capped: boolean },
+        deploy: () => Promise<EventsDeployOutcome>,
+    ): Promise<EventsDeployOutcome>;
+}

src/backend/drivers/ai-image/ImageGenerationDriver.test.ts

@@ -463,10 +463,26 @@ describe('ImageGenerationDriver.generate provider routing', () => {
             driver.generate({ prompt: 'a landscape' }),
         );
         expect(openaiImagesGenerateMock).toHaveBeenCalledWith(
-            expect.objectContaining({ model: 'gpt-image-2' }),
+            expect.objectContaining({ model: 'gpt-image-2.6-flare' }),
         );
     });
 
+    it.each(['gpt-image-2.6-flare', 'gpt-image-2.6-sunburst'])(
+        'routes %s and its OpenAI alias', async (model) => {
+            for (const id of [model, `openai/${model}`]) {
+                openaiImagesGenerateMock.mockResolvedValueOnce({
+                    data: [{ url: 'https://oai/image.png' }],
+                });
+                await withDriverName('ai-image', () => driver.generate({
+                    model: id, prompt: 'a landscape',
+                }));
+                expect(openaiImagesGenerateMock).toHaveBeenLastCalledWith(
+                    expect.objectContaining({ model }),
+                );
+            }
+        },
+    );
+
     it('rejects an excluded model regardless of caller casing', async () => {
         await expect(withActor(() =>
             driver.generate({

src/backend/drivers/ai-image/providers/openai/OpenAiImageProvider.test.ts

@@ -150,9 +150,9 @@ describe('OpenAiImageProvider construction', () => {
 // ── Model catalog ───────────────────────────────────────────────────
 
 describe('OpenAiImageProvider model catalog', () => {
-    it('returns gpt-image-2 as the default', () => {
+    it('returns gpt-image-2.6-flare as the default', () => {
         const provider = makeProvider();
-        expect(provider.getDefaultModel()).toBe('gpt-image-2');
+        expect(provider.getDefaultModel()).toBe('gpt-image-2.6-flare');
     });
 
     it('drops the deprecated gpt-image-1 family and chatgpt-image-latest', () => {
@@ -235,6 +235,56 @@ describe('OpenAiImageProvider.generate argument validation', () => {
 // ── Credit gate ─────────────────────────────────────────────────────
 
 describe('OpenAiImageProvider.generate credit gate', () => {
+    it('waits for the real usage write before returning an image', async () => {
+        const actor = makeActor({
+            user: {
+                id: 62,
+                uuid: `image-write-${Date.now()}`,
+                username: 'metered',
+            },
+        });
+        const redis = server.clients.redis as unknown as {
+            meterIncr: (...args: string[]) => Promise<unknown>;
+        };
+        const realIncr = redis.meterIncr.bind(redis);
+        let release!: () => void;
+        const held = new Promise<void>((resolve) => {
+            release = resolve;
+        });
+        let blocked = false;
+        vi.spyOn(redis, 'meterIncr').mockImplementation(async (...args) => {
+            if (!blocked && args[5].includes(`:actor:${actor.user!.uuid}:`)) {
+                blocked = true;
+                await held;
+            }
+            return realIncr(...args);
+        });
+        generateMock.mockResolvedValueOnce({
+            data: [{ url: 'https://image.example/result.png' }],
+        });
+        const provider = makeProvider();
+        let finished = false;
+        const generating = withTestActor(
+            () =>
+                provider.generate({
+                    model: 'gpt-image-2',
+                    prompt: 'an image',
+                    ratio: { w: 1024, h: 1024 },
+                }),
+            actor,
+        ).then((result) => {
+            finished = true;
+            return result;
+        });
+        try {
+            await vi.waitFor(() => expect(blocked).toBe(true));
+            expect(finished).toBe(false);
+        } finally {
+            release();
+            await generating;
+        }
+    });
+
     it('throws 402 BEFORE hitting OpenAI when actor lacks credits', async () => {
         const provider = makeProvider();
         hasCreditsSpy.mockResolvedValueOnce(false);
@@ -857,3 +907,64 @@ it('keeps a landscape aspect hint landscape at gpt-image-2 sizes', async () => {
     expect(w).toBeGreaterThan(h);
     expect(Math.abs(w / h - 1.7)).toBeLessThan(0.1);
 });
+
+
+describe('OpenAiImageProvider GPT Image 2.6', () => {
+    it.each(['gpt-image-2.6-flare', 'gpt-image-2.6-sunburst'])(
+        '%s generates and edits with token-based billing', async (model) => {
+            const provider = makeProvider();
+            const response = {
+                data: [{ b64_json: 'AAAA' }],
+                usage: {
+                    input_tokens: 100,
+                    output_tokens: 800,
+                    input_tokens_details: { text_tokens: 100, image_tokens: 0 },
+                },
+            };
+            generateMock.mockResolvedValue(response);
+            editMock.mockResolvedValue(response);
+            for (const input_image of [undefined, 'data:image/png;base64,AAAA']) {
+                const result = await withTestActor(() => provider.generate({
+                    model, prompt: 'hi', quality: 'xhigh',
+                    ratio: { w: 1024, h: 1024 }, input_image,
+                }));
+                expect(result).toBe('data:image/png;base64,AAAA');
+            }
+            for (const mock of [generateMock, editMock]) {
+                expect(mock.mock.calls[0]![0]).toMatchObject({
+                    model, quality: 'xhigh', size: '1024x1024',
+                });
+            }
+            const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!;
+            expect(entries).toEqual(expect.arrayContaining([
+                expect.objectContaining({
+                    usageType: `openai:${model}:xhigh:1024x1024:output`,
+                    costOverride: 2_400_000,
+                }),
+            ]));
+        },
+    );
+
+    it.each([
+        ['low', 1024, 1024, 823],
+        ['medium', 1024, 1024, 1449],
+        ['high', 1024, 1024, 1756],
+        ['xhigh', 1024, 1024, 3122],
+        ['max', 1024, 1024, 7024],
+        ['auto', 1024, 1024, 1449],
+        ['low', 2048, 1088, 571],
+    ])('estimates %s at %sx%s from the 2.6 calculator', async (quality, w, h, tokens) => {
+        generateMock.mockResolvedValue({ data: [{ b64_json: 'AAAA' }] });
+        await withTestActor(() => makeProvider().generate({
+            model: 'gpt-image-2.6-flare', prompt: 'hi', quality: String(quality),
+            ratio: { w: Number(w), h: Number(h) },
+        }));
+        const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!;
+        const output = (entries as Array<{ usageType: string; costOverride: number }>)
+            .find((entry) => entry.usageType.endsWith(':output'))!;
+        expect(output.usageType).toBe(`openai:gpt-image-2.6-flare:${quality}:${w}x${h}:output`);
+        // The metering conversion rounds fractional microcents upward.
+        expect(output.costOverride).toBeGreaterThanOrEqual(Number(tokens) * 3000);
+        expect(output.costOverride).toBeLessThanOrEqual(Number(tokens) * 3000 + 1);
+    });
+});

src/backend/drivers/ai-image/providers/openai/OpenAiImageProvider.ts

@@ -56,8 +56,8 @@ interface OpenAIImageUsage {
 
 /**
  * OpenAI image generation provider for v2. Supports the GPT Image models
- * (gpt-image-2, -2.5-sunburst, -2.5-flare), including image-to-image editing
- * via `input_images` (the `images.edit` endpoint).
+ * (gpt-image-2 and the 2.5/2.6 Sunburst and Flare models), including
+ * image-to-image editing via `input_images` (the `images.edit` endpoint).
  */
 export class OpenAiImageProvider implements IImageProvider {
     #meteringService: AiMeteringService;
@@ -109,7 +109,7 @@ export class OpenAiImageProvider implements IImageProvider {
     }
 
     getDefaultModel(): string {
-        return 'gpt-image-2';
+        return 'gpt-image-2.6-flare';
     }
 
     async generate({
@@ -293,7 +293,10 @@ export class OpenAiImageProvider implements IImageProvider {
                 });
             }
             if (usageEntries.length) {
-                this.#meteringService.batchIncrementUsages(actor, usageEntries);
+                await this.#meteringService.batchIncrementUsages(
+                    actor,
+                    usageEntries,
+                );
             }
 
             const url =
@@ -601,17 +604,36 @@ export class OpenAiImageProvider implements IImageProvider {
         height: number,
         quality?: string,
     ): number {
-        const factors = model.startsWith('gpt-image-2.5')
-            ? OpenAiImageProvider.#GPT_IMAGE_2_5_LATENT_FACTORS
-            : OpenAiImageProvider.#GPT_IMAGE_2_LATENT_FACTORS;
+        const isGptImage26 = model.startsWith('gpt-image-2.6');
+        const factors =
+            isGptImage26 || model.startsWith('gpt-image-2.5')
+                ? OpenAiImageProvider.#GPT_IMAGE_2_5_LATENT_FACTORS
+                : OpenAiImageProvider.#GPT_IMAGE_2_LATENT_FACTORS;
         const factor = factors[quality ?? ''] ?? factors.medium;
         const longEdge = Math.max(width, height);
         const shortEdge = Math.min(width, height);
-        const shortLatent = Math.round((factor * shortEdge) / longEdge);
+        const shortGrid = (factor * shortEdge) / longEdge;
+        const floorGrid = Math.floor(shortGrid);
+        const shortLatent =
+            isGptImage26 && shortGrid - floorGrid === 0.5
+                ? floorGrid + (floorGrid % 2)
+                : Math.round(shortGrid);
         const latentW = width >= height ? factor : shortLatent;
         const latentH = width >= height ? shortLatent : factor;
         const baseArea = latentW * latentH;
-        return Math.ceil((baseArea * (2_000_000 + width * height)) / 4_000_000);
+        const tokens = Math.ceil(
+            (baseArea * (2_000_000 + width * height)) / 4_000_000,
+        );
+        // GPT Image 2.6 bills more tokens at low and medium quality.
+        const estimatedQuality =
+            quality && quality in factors ? quality : 'medium';
+        const multiplier =
+            estimatedQuality === 'low'
+                ? 4.2
+                : estimatedQuality === 'medium'
+                  ? 3.3
+                  : 1;
+        return isGptImage26 ? Math.round(tokens * multiplier) : tokens;
     }
 
     #estimateOutputCostFromTokens(

src/backend/drivers/ai-image/providers/openai/models.ts

@@ -20,6 +20,46 @@
 import type { IImageModel } from '../../types.js';
 
 export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [
+    {
+        puterId: 'openai:openai/gpt-image-2.6-sunburst',
+        id: 'gpt-image-2.6-sunburst',
+        aliases: ['openai/gpt-image-2.6-sunburst'],
+        name: 'GPT Image 2.6 Sunburst',
+        version: '2.6',
+        costs_currency: 'usd-cents',
+        index_cost_key: 'low:1024x1024',
+        costs: {
+            // Text tokens (per 1M tokens)
+            text_input: 500, // $5.00
+            text_cached_input: 125, // $1.25
+            // Image tokens (per 1M tokens)
+            image_input: 800, // $8.00
+            image_cached_input: 200, // $2.00
+            image_output: 3000, // $30.00
+            'low:1024x1024': 2.469,
+        },
+        allowedQualityLevels: ['low', 'medium', 'high', 'xhigh', 'max', 'auto'],
+    },
+    {
+        puterId: 'openai:openai/gpt-image-2.6-flare',
+        id: 'gpt-image-2.6-flare',
+        aliases: ['openai/gpt-image-2.6-flare'],
+        name: 'GPT Image 2.6 Flare',
+        version: '2.6',
+        costs_currency: 'usd-cents',
+        index_cost_key: 'low:1024x1024',
+        costs: {
+            // Text tokens (per 1M tokens)
+            text_input: 500, // $5.00
+            text_cached_input: 125, // $1.25
+            // Image tokens (per 1M tokens)
+            image_input: 800, // $8.00
+            image_cached_input: 200, // $2.00
+            image_output: 3000, // $30.00
+            'low:1024x1024': 2.469,
+        },
+        allowedQualityLevels: ['low', 'medium', 'high', 'xhigh', 'max', 'auto'],
+    },
     {
         puterId: 'openai:openai/gpt-image-2.5-sunburst',
         id: 'gpt-image-2.5-sunburst',

src/backend/drivers/apps/AppDriver.js

@@ -31,7 +31,7 @@ import {
     normalizeRawBase64ImageString,
     validateIconDataUrl,
 } from '../../util/appIcon.js';
-import { toAppReadView } from '../../util/appView.js';
+import { isOriginBootstrapApp, toAppReadView } from '../../util/appView.js';
 import { awaitLater } from '../../util/concurrency.js';
 import { isUniqueViolation } from '../../util/dbError.js';
 import {
@@ -1088,22 +1088,6 @@ export class AppDriver extends PuterDriver {
         }
     }
 
-    /**
-     * Origin-bootstrap detection: rows auto-created when an unknown origin
-     * first needed an app row (no human-supplied metadata). Marker is `name ===
-     * uid && title === uid` and a description starting with "App created from
-     * origin ". Only these rows are eligible for same-owner merging — refusing
-     * to merge arbitrary same-owner apps prevents accidental data loss.
-     */
-    #isOriginBootstrapApp(app) {
-        if (!app || typeof app !== 'object') return false;
-        if (typeof app.uid !== 'string' || !app.uid) return false;
-        if (app.name !== app.uid) return false;
-        if (app.title !== app.uid) return false;
-        if (typeof app.description !== 'string') return false;
-        return app.description.startsWith('App created from origin ');
-    }
-
     // -- Canonical-uid alias kvstore pair -----------------------------
     //
     // After a merge, the source app uid → canonical uid mapping is
@@ -1266,7 +1250,7 @@ export class AppDriver extends PuterDriver {
         if (
             Number.isInteger(conflictOwnerUserId) &&
             conflictOwnerUserId === user.id &&
-            !this.#isOriginBootstrapApp(appToJoin)
+            !isOriginBootstrapApp(appToJoin)
         ) {
             // Prevent merging arbitrary same-owner apps; only allow the
             // auto-created origin bootstrap row to be absorbed.

src/backend/drivers/notification/NotificationDriver.appMessage.test.ts

@@ -188,7 +188,11 @@ describe('app.message with live delivery over the socket wire', () => {
             captureBus(state.server, 'outer.gui.notif.message', async () => {
                 created = (await withActor(asApp(holder, app), () =>
                     state.driver.create(
-                        message({ text: 'Body', channel: 'replies' }),
+                        message({
+                            text: 'Body',
+                            priority: 'high',
+                            channel: 'replies',
+                        }),
                     ),
                 )) as Record<string, unknown>;
             }).then((seen) => (desktop = seen)),
@@ -200,7 +204,7 @@ describe('app.message with live delivery over the socket wire', () => {
             title: 'Hello',
             text: 'Body',
             channel: 'replies',
-            priority: 'normal',
+            priority: 'high',
             ttl: 3600,
             app: { uid: app.uid, name: 'App message test' },
         });
@@ -212,6 +216,7 @@ describe('app.message with live delivery over the socket wire', () => {
             app_uid: app.uid,
             audience: 'app-user',
             type: 'app.message',
+            value: expect.objectContaining({ priority: 'high' }),
         });
         // Not silent, but sent for the app alone: the desktop wire stays quiet.
         expect(written).toEqual([
@@ -220,7 +225,7 @@ describe('app.message with live delivery over the socket wire', () => {
                 uid,
                 type: 'app.message',
                 appUid: app.uid,
-            }),
+                value: expect.objectContaining({ priority: 'high' }),            }),
         ]);
         expect(desktop).toEqual([]);
     });

src/backend/drivers/notification/NotificationDriver.handle.integration.test.ts

@@ -207,6 +207,7 @@ describe('app.message to the other party of a share', () => {
         const written = await captureBus('notif.written', async () => {
             sent = await send(inApp(owner, app), {
                 title: 'New message',
+                priority: 'high',
                 to: { handle },
             });
         });
@@ -222,7 +223,7 @@ describe('app.message to the other party of a share', () => {
         expect(row!.value).toEqual({
             type: 'app.message',
             title: 'New message',
-            priority: 'normal',
+            priority: 'high',
             ttl: 3600,
             app: { uid: app.uid, name: 'Handle App' },
         });

src/backend/drivers/notification/NotificationDriver.sendAt.test.ts

@@ -0,0 +1,506 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * `notify({ sendAt })` without and with `services.schedules`, and the deferred
+ * mailbox write an extension runs inside its own transaction, fenced on a guard
+ * in a table of its own (a test table here).
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { runWithContext } from '../../core/context.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    DeferredSendPlan,
+    ScheduleServiceContract,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { EVENTS_BACKGROUND_PERMISSION } from '../../services/events/authorization.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import type {
+    DeferredWriteTx,
+    NotificationDriver,
+} from './NotificationDriver.js';
+
+vi.setConfig({ testTimeout: 120_000 });
+
+const BOOT_TIMEOUT_MS = 480_000;
+const TEST_TABLE = 'deferred_send_test';
+
+let server: PuterServer;
+let holder: Actor;
+let app: { id: number; uid: string };
+let saved: unknown;
+let nextId = 1;
+
+const db = () => server.clients.db;
+const driver = () =>
+    server.drivers.notifications as unknown as NotificationDriver;
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+const asApp = (): Actor => makeActor({ user: holder.user, app });
+
+const notify = (fields: Record<string, unknown>): Promise<unknown> =>
+    runWithContext({ actor: asApp() }, () =>
+        driver().create({
+            object: { value: { type: 'app.message', title: 'Hi', ...fields } },
+        }),
+    );
+
+const codeOf = async (run: Promise<unknown>): Promise<string | undefined> => {
+    try {
+        await run;
+    } catch (err) {
+        if (isHttpError(err)) return err.legacyCode;
+        throw err;
+    }
+    return undefined;
+};
+
+const mailbox = () =>
+    db().read(
+        'SELECT * FROM `notification` WHERE `user_id` = ? AND `app_uid` = ? ' +
+            'ORDER BY `id`',
+        [holder.user!.id, app.uid],
+    );
+
+/** A row's value: JSON text on sqlite, parsed on the other engines. */
+const valueOf = (row: Record<string, unknown>) =>
+    typeof row.value === 'string' ? JSON.parse(row.value) : row.value;
+
+/** A send claimed under `token`, as an extension's own row would be. */
+const claimed = async (token: string): Promise<number> => {
+    const id = nextId++;
+    await db().write(
+        `INSERT INTO \`${TEST_TABLE}\` (\`id\`, \`token\`, \`state\`) VALUES (?, ?, 'sending')`,
+        [id, token],
+    );
+    return id;
+};
+
+/** The transaction an extension wraps the write in: its row, then the mailbox. */
+const txFor = (id: number, token: string): DeferredWriteTx => ({
+    guard: {
+        table: TEST_TABLE,
+        where: "`id` = ? AND `token` = ? AND `state` = 'delivered'",
+        values: [id, token],
+    },
+    async commit(entries: readonly BatchEntry[]) {
+        await db().batchWrite([
+            {
+                statement:
+                    `UPDATE \`${TEST_TABLE}\` SET \`state\` = 'delivered' ` +
+                    "WHERE `id` = ? AND `token` = ? AND `state` = 'sending'",
+                values: [id, token],
+            },
+            ...entries,
+        ]);
+        const [row] = await db().pread(
+            `SELECT \`state\`, \`token\` FROM \`${TEST_TABLE}\` WHERE \`id\` = ?`,
+            [id],
+        );
+        return row?.state === 'delivered' && row.token === token;
+    },
+});
+
+const write = (
+    sendKey: string,
+    tx: DeferredWriteTx,
+    over: {
+        payload?: string | null;
+        appUid?: string;
+        signal?: AbortSignal;
+    } = {},
+) =>
+    driver().deferredAppMessages.write(
+        {
+            holderUserId: holder.user!.id,
+            appUid: over.appUid ?? app.uid,
+            payload:
+                over.payload === undefined
+                    ? JSON.stringify({ message: { title: 'Later' } })
+                    : over.payload,
+            sendKey,
+            signal: over.signal ?? new AbortController().signal,
+        },
+        tx,
+    );
+
+const fakeSchedules = (plan: Partial<DeferredSendPlan>) =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(),
+        list: vi.fn(),
+        get: vi.fn(),
+        update: vi.fn(),
+        remove: vi.fn(),
+        receivePeerChanges: vi.fn(),
+        prepareSendAt: vi.fn(async (): Promise<DeferredSendPlan> => ({
+            sendAt: 0,
+            immediate: false,
+            holderUserId: holder.user!.id,
+            appUid: app.uid,
+            handlerDepth: 0,
+            quota: {
+                recurringPerUser: 1,
+                recurringPerApp: 1,
+                oneShotsPerUser: 1,
+                oneShotsPerApp: 1,
+                budgetPerUser: 1,
+                budgetPerApp: 1,
+            },
+            ...plan,
+        })),
+        createDeferredAppMessage: vi.fn(async (p: DeferredSendPlan) => ({
+            scheduleId: 'sched-1',
+            sendAt: p.sendAt,
+        })),
+    }) satisfies ScheduleServiceContract;
+
+beforeAll(async () => {
+    server = await setupTestServer({
+        events: { enabled: true },
+    } as IConfig);
+    const username = `sendat-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+        requires_email_confirmation: false,
+    });
+    const row = (await server.stores.user.getById(user.id))!;
+    holder = makeActor({
+        user: {
+            id: row.id,
+            uuid: row.uuid,
+            username: row.username,
+            email: row.email ?? null,
+            email_confirmed: true,
+        } as Actor['user'],
+    });
+    const name = `sendat-app-${Math.random().toString(36).slice(2, 10)}`;
+    app = (await server.stores.app.create(
+        { name, title: name, index_url: `https://${name}.test/` },
+        { ownerUserId: row.id },
+    )) as { id: number; uid: string };
+    await server.services.permission.grantUserAppPermission(
+        holder,
+        app.uid,
+        EVENTS_BACKGROUND_PERMISSION,
+    );
+    await db().write(
+        `CREATE TABLE \`${TEST_TABLE}\` (\`id\` INTEGER PRIMARY KEY, ` +
+            '`token` VARCHAR(64) NOT NULL, `state` VARCHAR(16) NOT NULL)',
+        [],
+    );
+    saved = slot().schedules;
+}, BOOT_TIMEOUT_MS);
+
+beforeEach(async () => {
+    await db().write('DELETE FROM `notification` WHERE `app_uid` = ?', [
+        app.uid,
+    ]);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+    vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+describe('notify({ sendAt }) without services.schedules', () => {
+    it('is refused events_disabled, even when due now, storing nothing', async () => {
+        setSchedules(undefined);
+        expect(await codeOf(notify({ sendAt: Date.now() }))).toBe(
+            'events_disabled',
+        );
+        expect(await codeOf(notify({ sendAt: Date.now() + 3_600_000 }))).toBe(
+            'events_disabled',
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends as before without sendAt', async () => {
+        setSchedules(undefined);
+        const sent = (await notify({ text: 'now' })) as { uid: string };
+        expect(sent.uid).toBeTruthy();
+        expect(await mailbox()).toHaveLength(1);
+    });
+});
+
+describe('notify({ sendAt }) with services.schedules', () => {
+    it('hands a later send to createDeferredAppMessage and answers its id', async () => {
+        const at = Date.now() + 3_600_000;
+        const schedules = fakeSchedules({ sendAt: at });
+        setSchedules(schedules);
+
+        await expect(
+            notify({ sendAt: at, text: 'later', priority: 'high' }),
+        ).resolves.toEqual({
+            scheduled: { scheduleId: 'sched-1', sendAt: at },
+        });
+        expect(schedules.prepareSendAt).toHaveBeenCalledWith(
+            expect.objectContaining({ user: holder.user }),
+            at,
+        );
+        expect(schedules.createDeferredAppMessage).toHaveBeenCalledWith(
+            expect.objectContaining({ sendAt: at }),
+            {
+                message: expect.objectContaining({
+                    title: 'Hi',
+                    text: 'later',
+                    priority: 'high',
+                }),
+            },
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends at once when the plan says it is due', async () => {
+        const schedules = fakeSchedules({ immediate: true });
+        setSchedules(schedules);
+
+        const sent = (await notify({ sendAt: Date.now() })) as {
+            uid: string;
+        };
+        expect(sent.uid).toBeTruthy();
+        expect(schedules.createDeferredAppMessage).not.toHaveBeenCalled();
+        expect(await mailbox()).toHaveLength(1);
+    });
+
+    it('passes a refusal from prepareSendAt through', async () => {
+        const schedules = fakeSchedules({});
+        schedules.prepareSendAt.mockRejectedValueOnce(
+            Object.assign(new Error('scoped'), {}),
+        );
+        setSchedules(schedules);
+        await expect(notify({ sendAt: Date.now() })).rejects.toThrow('scoped');
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+describe('deferredAppMessages.write', () => {
+    it('writes one row keyed by the send, in the caller`s transaction', async () => {
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(
+            write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', priority: 'high' },
+                }),
+            }),
+        ).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: key, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({
+            title: 'Later',
+            priority: 'high',
+        });
+    });
+
+    it('writes nothing when the guard no longer holds: another claim took it', async () => {
+        const id = await claimed(uuidv4());
+        const key = uuidv4();
+
+        await expect(write(key, txFor(id, uuidv4()))).resolves.toBe('lost');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('replaces the open row of its group, which takes the key', async () => {
+        const open = (await notify({ group: 'g', text: 'first' })) as {
+            uid: string;
+        };
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(
+            write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g' },
+                }),
+            }),
+        ).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: open.uid, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
+    });
+
+    it('drops a payload it can`t read, and a sender whose app is gone', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        await expect(
+            write(uuidv4(), txFor(id, token), { payload: '{"nope":1}' }),
+        ).resolves.toBe('dropped');
+        await expect(
+            write(uuidv4(), txFor(id, token), { appUid: `app-${uuidv4()}` }),
+        ).resolves.toBe('dropped');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('writes nothing once the caller`s claim is lost', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        const lost = new AbortController();
+        lost.abort();
+
+        await expect(
+            write(uuidv4(), txFor(id, token), { signal: lost.signal }),
+        ).rejects.toThrow();
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+/**
+ * A second claim taking a send over while the first one's transaction is open.
+ * Needs two writers at once with row locks; sqlite serialises writers, so this
+ * runs on postgres only.
+ */
+describe.runIf(process.env.PUTER_TEST_DB_ENGINE === 'postgres')(
+    'a deferred write racing another writer',
+    () => {
+        /**
+         * Hold the first statement matching `match` on its connection, before
+         * or after it runs, until released.
+         */
+        const holdStatement = (match: RegExp, when: 'before' | 'after') => {
+            const client = db() as unknown as {
+                query: (
+                    target: unknown,
+                    sql: string,
+                    params?: unknown[],
+                ) => Promise<unknown>;
+            };
+            const original = client.query.bind(client);
+            let reached!: () => void;
+            let release!: () => void;
+            const atBarrier = new Promise<void>((r) => (reached = r));
+            const released = new Promise<void>((r) => (release = r));
+            let held = false;
+            vi.spyOn(client as never, 'query' as never).mockImplementation(
+                (async (target: unknown, sql: string, params?: unknown[]) => {
+                    if (held || !match.test(sql))
+                        return original(target, sql, params);
+                    held = true;
+                    if (when === 'before') {
+                        reached();
+                        await released;
+                        return original(target, sql, params);
+                    }
+                    const result = await original(target, sql, params);
+                    reached();
+                    await released;
+                    return result;
+                }) as never,
+            );
+            return { atBarrier, release };
+        };
+
+        const DELIVERED = /SET `state` = 'delivered'/;
+        const REPLACE = /`replaced_at` = \?, `send_key` = \? WHERE/;
+
+        /** Whether `promise` is still pending after `ms`. */
+        const stillPending = async (promise: Promise<unknown>, ms: number) => {
+            let settled = false;
+            const watched = (async () => {
+                try {
+                    await promise;
+                } finally {
+                    settled = true;
+                }
+            })();
+            await new Promise((r) => setTimeout(r, ms));
+            void watched;
+            return !settled;
+        };
+
+        it('a transaction that opened before a takeover writes nothing', async () => {
+            const stale = uuidv4();
+            const id = await claimed(stale);
+            const key = uuidv4();
+            const gate = holdStatement(DELIVERED, 'before');
+            const first = write(key, txFor(id, stale));
+            await gate.atBarrier;
+
+            await db().write(
+                `UPDATE \`${TEST_TABLE}\` SET \`token\` = ? WHERE \`id\` = ?`,
+                [uuidv4(), id],
+            );
+            gate.release();
+            await expect(first).resolves.toBe('lost');
+            expect(
+                await server.stores.notification.getBySendKey(key),
+            ).toBeNull();
+            expect(await mailbox()).toEqual([]);
+        });
+
+        it('an immediate replacement waiting on the open transaction lands after it, without the key', async () => {
+            const open = (await notify({ group: 'g', text: 'first' })) as {
+                uid: string;
+            };
+            const token = uuidv4();
+            const id = await claimed(token);
+            const key = uuidv4();
+            const gate = holdStatement(REPLACE, 'after');
+            const deferred = write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g', text: 'later' },
+                }),
+            });
+            await gate.atBarrier;
+
+            const immediate = notify({ group: 'g', text: 'newer' });
+            // Past the group lock's wait, then blocked on the row.
+            expect(await stillPending(immediate, 2_500)).toBe(true);
+            gate.release();
+            await expect(deferred).resolves.toBe('delivered');
+            await immediate;
+
+            const rows = await mailbox();
+            expect(rows).toHaveLength(1);
+            expect(rows[0]).toMatchObject({ uid: open.uid, send_key: null });
+            expect(valueOf(rows[0]).text).toBe('newer');
+        });
+    },
+);

src/backend/drivers/notification/NotificationDriver.test.ts

@@ -859,3 +859,32 @@ describe('NotificationDriver — rows written before the scope columns', () => {
         expect(await selectUids(holder.actor)).toEqual([uid]);
     });
 });
+
+
+describe('NotificationDriver displayed notification catch-up', () => {
+    it('selects named ids within the app and user scope', async () => {
+        const box = await mailbox();
+        await server.services.notification.markAcknowledged(box.rows.appUser, box.holder.userId);
+        const rows = await withActor(box.ownAppActor, () => driver.select({
+            subject: 'notif:app-user',
+            uids: Object.values(box.rows),
+            limit: 50,
+        }));
+        expect(rows).toEqual([expect.objectContaining({
+            uid: box.rows.appUser, acknowledged: expect.any(Number),
+        })]);
+    });
+
+    it.each([Array(51).fill('one'), [''], [7], 'one', ['x'.repeat(129)]])(
+        'rejects an invalid or oversized id batch: %s', async (uids) => {
+            const { actor } = await makeUser();
+            await expect(withActor(actor, () => driver.select({ uids })))
+                .rejects.toMatchObject({ statusCode: 400 });
+        },
+    );
+
+    it('returns no rows for an empty id batch', async () => {
+        const { actor } = await makeUser();
+        expect(await withActor(actor, () => driver.select({ uids: [] }))).toEqual([]);
+    });
+});

src/backend/drivers/notification/NotificationDriver.ts

@@ -46,11 +46,19 @@ import {
     ownedAppUids,
 } from '../../services/notification/notificationAudience.js';
 import {
-    NOTIFICATION_VALUE_MAX_BYTES,
+    assertNotificationValueFits,
     OUTSIDE_ACCOUNT_MAILBOX_TYPES,
     resolveNotificationWrite,
     type NotificationTypeName,
 } from '../../services/notification/notificationTypes.js';
+import {
+    schedulesUnavailable,
+    type DeferredAppMessage,
+    type DeferredSendPlan,
+    type ScheduleServiceContract,
+    type SqlGuard,
+} from '../../core/schedules.js';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
 import { withRedisLock } from '../../util/redisLock.js';
 import { PuterDriver } from '../types.js';
 import type { Actor } from '../../core/actor.js';
@@ -67,6 +75,44 @@ const GROUP_LOCK_RETRY_MS = 25;
 
 type AppMessageWindow = 'minute' | 'day';
 
+export type { SqlGuard };
+
+/** What a delayed app message's fire-time write needs from its row. */
+export interface DeferredWriteInput {
+    holderUserId: number;
+    appUid: string;
+    /** The stored {@link DeferredAppMessage}, as JSON text. */
+    payload: string | null;
+    /** The slot's key: the written row's uid or `send_key`, and its `sendId`. */
+    sendKey: string;
+    /** Aborted when the caller's claim is lost. */
+    signal: AbortSignal;
+}
+
+/**
+ * The caller's transaction the mailbox write commits in. Atomicity is the
+ * caller's to keep; the type can't enforce it.
+ */
+export interface DeferredWriteTx {
+    /**
+     * What every mailbox statement is fenced on: the caller's row marked
+     * delivered by this claim.
+     */
+    guard: SqlGuard;
+    /**
+     * Must be the whole delivery in one step: the caller's guarded transition
+     * of its row to delivered and `entries`, in that order, in one
+     * `batchWrite`, then a read of the row on the primary. True only when that
+     * read shows this claim holding the row as delivered. Splitting the
+     * transition from `entries` lets one slot write two rows.
+     */
+    commit(entries: readonly BatchEntry[]): Promise<boolean>;
+}
+
+/** How a delayed send's write stood when it stopped. */
+export type DeferredWriteOutcome =
+    'delivered' | 'dropped' | 'lost' | 'senderSuspended';
+
 /** The mailbox slice a call named, in the terms the query takes it. */
 interface MailboxScope {
     audiences: readonly string[];
@@ -74,19 +120,6 @@ interface MailboxScope {
     appUid: string | null | undefined;
 }
 
-const assertValueFits = (value: Record<string, unknown>): void => {
-    if (
-        Buffer.byteLength(JSON.stringify(value), 'utf8') >
-        NOTIFICATION_VALUE_MAX_BYTES
-    ) {
-        throw new HttpError(
-            400,
-            `\`value\` may not exceed ${NOTIFICATION_VALUE_MAX_BYTES} bytes`,
-            { legacyCode: 'notification_value_too_large' },
-        );
-    }
-};
-
 const typeNotAllowed = (message: string): HttpError =>
     new HttpError(403, message, {
         legacyCode: 'forbidden',
@@ -124,6 +157,21 @@ const parseNotifyTarget = (to: unknown): string => {
     return target.handle;
 };
 
+/** A stored delayed message, or `null` for one that can't be read back. */
+const parseDelayedAppMessage = (
+    payload: string | null,
+): DeferredAppMessage | null => {
+    try {
+        const stored = JSON.parse(payload ?? 'null') as DeferredAppMessage;
+        if (typeof stored?.message?.title !== 'string') return null;
+        if (stored.to !== undefined && typeof stored.to !== 'string')
+            return null;
+        return stored;
+    } catch {
+        return null;
+    }
+};
+
 /** Whether a row belongs to the slice, as the same SQL scope selected it. */
 const inScope = (
     row: Record<string, unknown>,
@@ -216,7 +264,7 @@ export class NotificationDriver extends PuterDriver {
                 value,
             );
         }
-        assertValueFits(value);
+        assertNotificationValueFits(value);
         if (type === APP_MESSAGE_TYPE) {
             throw typeNotAllowed(`Only apps can create ${APP_MESSAGE_TYPE}`);
         }
@@ -281,13 +329,34 @@ export class NotificationDriver extends PuterDriver {
             : predicate;
 
         const scope = this.#resolveScope(actor, args.subject);
+        const uids = args.uids;
+        if (
+            uids !== undefined &&
+            (!Array.isArray(uids) ||
+                uids.length > 50 ||
+                uids.some(
+                    (uid) =>
+                        typeof uid !== 'string' ||
+                        uid.length === 0 ||
+                        uid.length > 128,
+                ))
+        ) {
+            throw new HttpError(
+                400,
+                'uids must contain at most 50 notification ids',
+                {
+                    legacyCode: 'bad_request',
+                },
+            );
+        }
         const query: {
             limit: number;
             scope: MailboxScope | null;
             filter?: string;
             onlyUnacknowledged?: boolean;
             excludeTypes?: readonly string[];
-        } = { limit, scope };
+            uids?: string[];
+        } = { limit, scope, ...(uids ? { uids: uids as string[] } : {}) };
         // The whole mailbox, or a slice naming no app, is the desktop's view of
         // it, which leaves out the types only an app has a use for. A slice
         // naming the app still returns them.
@@ -354,7 +423,7 @@ export class NotificationDriver extends PuterDriver {
         object: Record<string, unknown>,
         value: Record<string, unknown>,
     ): Promise<unknown> {
-        const { type, to, ...fields } = value;
+        const { type, to, sendAt, ...fields } = value;
         if (type !== APP_MESSAGE_TYPE) {
             throw typeNotAllowed(`Apps can create only ${APP_MESSAGE_TYPE}`);
         }
@@ -369,7 +438,13 @@ export class NotificationDriver extends PuterDriver {
         }
         const handle = to === undefined ? undefined : parseNotifyTarget(to);
         const message = parseAppMessage(fields);
-        assertValueFits(value);
+        assertNotificationValueFits(value);
+        const timing =
+            sendAt === undefined || sendAt === null
+                ? null
+                : await this.#schedules().prepareSendAt(actor, sendAt);
+        if (timing && !timing.immediate)
+            return this.#scheduleAppMessage(actor, message, handle, timing);
 
         if (handle === undefined) {
             const uid = await this.#sendAppMessage(
@@ -410,12 +485,23 @@ export class NotificationDriver extends PuterDriver {
         appUid: string,
         message: AppMessage,
     ): Promise<string> {
-        const send = () =>
-            this.#replaceOrWriteAppMessage(recipientId, appUid, message);
+        return this.#oneSendPerGroup(recipientId, appUid, message, () =>
+            this.#replaceOrWriteAppMessage(recipientId, appUid, message),
+        );
+    }
+
+    /**
+     * One send per group at a time, so a burst into an empty group writes one
+     * row that the rest replace. A lock that is busy past the wait or
+     * unreachable still sends: the race costs a second row and its day slot.
+     */
+    #oneSendPerGroup<T>(
+        recipientId: number,
+        appUid: string,
+        message: AppMessage,
+        send: () => Promise<T>,
+    ): Promise<T> {
         if (!message.group) return send();
-        // One send per group at a time, so a burst into an empty group writes
-        // one row that the rest replace. A lock that is busy past the wait or
-        // unreachable still sends: the race costs a second row and its day slot.
         return withRedisLock(
             this.clients.redis,
             `notif:app-message:group:${recipientId}:${appUid}:${message.group}`,
@@ -430,6 +516,161 @@ export class NotificationDriver extends PuterDriver {
         );
     }
 
+    // -- Delayed sends -----------------------------------------------
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /**
+     * Store an app's message to go out at `sendAt`. A recipient named through a
+     * share is checked now without spending its budget; every limit is spent
+     * when it goes, and the share is checked again then.
+     */
+    async #scheduleAppMessage(
+        actor: UserActor,
+        message: AppMessage,
+        handle: string | undefined,
+        plan: DeferredSendPlan,
+    ): Promise<{ scheduled: { scheduleId: string; sendAt: number } }> {
+        if (handle !== undefined)
+            await this.services.kvShare.resolveNotifyRecipient(actor, handle, {
+                spendBudget: false,
+            });
+        return {
+            scheduled: await this.#schedules().createDeferredAppMessage(
+                plan,
+                handle === undefined ? { message } : { message, to: handle },
+            ),
+        };
+    }
+
+    /** For extensions. A field, not a method, so `/drivers/call` can't reach it. */
+    readonly deferredAppMessages = {
+        write: (input: DeferredWriteInput, tx: DeferredWriteTx) =>
+            this.#writeDeferredAppMessage(input, tx),
+    };
+
+    /**
+     * Check the sender and recipient as they stand now, spend the recipient's
+     * limits, and write the mailbox row through `tx`, fenced on its guard.
+     * Refusals are the same `HttpError`s an immediate send throws.
+     */
+    async #writeDeferredAppMessage(
+        input: DeferredWriteInput,
+        tx: DeferredWriteTx,
+    ): Promise<DeferredWriteOutcome> {
+        const { holderUserId, sendKey, signal } = input;
+        const sender = await this.services.events.resolveScheduleActor(
+            holderUserId,
+            input.appUid,
+        );
+        if (!sender) {
+            // A lagging replica can miss an app the primary has: retried.
+            const there =
+                (await this.stores.app.uidExistsOnPrimary(input.appUid)) &&
+                (await this.stores.user.getById(holderUserId)) !== null;
+            if (there) throw new Error('the sender could not be resolved');
+            return 'dropped';
+        }
+        if (sender.user?.suspended) return 'senderSuspended';
+        // Uninstalled since, and the cancel at uninstall missed it.
+        const appId = sender.effectiveApp?.id;
+        if (
+            typeof appId !== 'number' ||
+            !(await this.stores.permission.hasAnyUserAppPerm(
+                holderUserId,
+                appId,
+            ))
+        )
+            return 'dropped';
+        const stored = parseDelayedAppMessage(input.payload);
+        if (!stored) return 'dropped';
+        const { message } = stored;
+
+        let recipientId = holderUserId;
+        let appUid = input.appUid;
+        if (stored.to !== undefined) {
+            const recipient =
+                await this.services.kvShare.resolveNotifyRecipient(
+                    sender,
+                    stored.to,
+                );
+            recipientId = recipient.recipientUserId;
+            appUid = recipient.appUid;
+        }
+
+        return this.#oneSendPerGroup(recipientId, appUid, message, async () => {
+            // Primary, as for an immediate send.
+            const open = message.group
+                ? await this.stores.notification.getUnseenByGroup(
+                      recipientId,
+                      appUid,
+                      message.group,
+                      { primary: true },
+                  )
+                : null;
+            await this.#spendAppMessageLimits(
+                recipientId,
+                appUid,
+                open ? ['minute'] : ['minute', 'day'],
+            );
+            message.app = appMessageSender(
+                appUid,
+                await this.stores.app.getByUid(appUid),
+            );
+            // Another attempt may hold the lease now; hand the claim back.
+            if (signal.aborted) throw new Error('delayed send lost its lease');
+
+            const registered = resolveNotificationWrite(
+                APP_MESSAGE_TYPE,
+                appUid,
+            );
+            const delivered = await tx.commit(
+                this.stores.notification.sendEntries(
+                    {
+                        sendKey,
+                        userId: recipientId,
+                        value: { ...message, type: registered.type },
+                        type: registered.type,
+                        audience: registered.audience,
+                        appUid,
+                        group: message.group ?? null,
+                        replaceUid: open ? String(open.uid) : null,
+                        at: Date.now(),
+                    },
+                    tx.guard,
+                ),
+            );
+            if (!delivered) return 'lost';
+            if (open)
+                await this.#spendDayIfFresh(recipientId, appUid, open, sendKey);
+            return 'delivered';
+        });
+    }
+
+    /**
+     * A replacement that found its row shown since the lookup wrote a new one,
+     * which costs a day slot. Already written, so a full day is not a refusal.
+     */
+    async #spendDayIfFresh(
+        recipientId: number,
+        appUid: string,
+        open: Record<string, unknown>,
+        sendKey: string,
+    ): Promise<void> {
+        try {
+            const written =
+                await this.stores.notification.getBySendKey(sendKey);
+            if (written && written.uid !== open.uid)
+                await this.#spendAppMessageLimits(recipientId, appUid, ['day']);
+        } catch {
+            // The row is written; a slot not spent is the only loss.
+        }
+    }
+
     async #replaceOrWriteAppMessage(
         recipientId: number,
         appUid: string,

src/backend/drivers/util/pipeStreamResult.test.ts

@@ -129,6 +129,27 @@ describe('pipeStreamResult', () => {
         expect(warn).not.toHaveBeenCalled();
     });
 
+    it('adds at most two close listeners to the response', async () => {
+        // The response already carries close listeners from metering,
+        // compression and tracing; Node warns past ten.
+        let added = -1;
+        server = createServer((_req, res) => {
+            const before = res.listenerCount('close');
+            pipeStreamResult(Readable.from(['a', 'b']), res, 'test.run');
+            added = res.listenerCount('close') - before;
+        });
+        await new Promise<void>((resolve) => server!.listen(0, resolve));
+        const { port } = server.address() as AddressInfo;
+        await new Promise<void>((resolve) =>
+            get(`http://127.0.0.1:${port}/`, (res) => {
+                res.resume();
+                res.on('end', () => resolve());
+            }),
+        );
+        expect(added).toBeGreaterThan(0);
+        expect(added).toBeLessThanOrEqual(2);
+    });
+
     it('drops the source, without throwing, when the client left before the result was ready', async () => {
         const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
         const source = Readable.from(['late']);

src/backend/drivers/util/pipeStreamResult.ts

@@ -17,7 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import { pipeline, type Readable } from 'node:stream';
+import type { Readable } from 'node:stream';
 
 // Only what's used here, so callers on another @types/node (or express's
 // Response) still fit.
@@ -32,24 +32,32 @@ type ResponseSink = NodeJS.WritableStream & {
  * mid-body destroys the response, so the client sees an aborted transfer rather
  * than a clean end, and the failure is logged once instead of surfacing as an
  * unhandled stream error. A client that hangs up stops the source.
+ *
+ * `pipe` rather than `pipeline`: pipeline puts seven close listeners on the
+ * response, and with the ones metering, compression and tracing add, every
+ * streamed response tripped Node's ten-listener warning.
  */
 export function pipeStreamResult(
     stream: Readable,
     res: ResponseSink,
     label: string,
 ): void {
-    // The caller hung up while the driver ran; `pipeline` throws on a closed
-    // destination, so drop the source instead.
+    // The caller hung up while the driver ran: nothing will read the source.
     if (res.closed || res.destroyed) {
         stream.destroy();
         return;
     }
-    pipeline(stream, res, (err) => {
-        if (!err) return;
+    stream.once('error', (err) => {
+        stream.unpipe(res);
         if (!res.destroyed) res.destroy();
-        // The client hanging up is not a failure of ours.
-        if ((err as { code?: string }).code === 'ERR_STREAM_PREMATURE_CLOSE')
-            return;
         console.warn(`[drivers] ${label} stream failed mid-response:`, err);
     });
+    // A response that closes before the source ended is a client hanging up.
+    res.once('close', () => {
+        if (stream.readableEnded) return;
+        stream.unpipe(res);
+        stream.destroy();
+    });
+    res.once('error', () => stream.destroy());
+    stream.pipe(res);
 }

src/backend/drivers/workers/WorkerDriver.cloudflare.test.ts

@@ -754,6 +754,50 @@ describe('WorkerDriver.create with a configured deploy backend', () => {
             expect(putCalls()).toHaveLength(1);
         });
 
+        const uploadedMetadata = () => {
+            const [, init] = putCalls().at(-1)!;
+            const form = (init as RequestInit).body as FormData;
+            return JSON.parse(form.get('metadata') as string);
+        };
+
+        it('sends limits.cpu_ms in the upload metadata when cpuMs is set', async () => {
+            const { user, actor } = await makeUser();
+            const path = `/${user.username}/cpu.worker.js`;
+            await writeSource(actor, user.id, path, 'export default {}');
+            const scriptName = `dt-cpu-${user.username}`;
+            await inCtx(actor, () =>
+                target.create({
+                    appId: '',
+                    workerName: scriptName,
+                    filePath: path,
+                    [INTERNAL_DEPLOY_TARGET]: { scriptName, cpuMs: 50 },
+                }),
+            );
+            expect(uploadedMetadata().limits).toEqual({ cpu_ms: 50 });
+        });
+
+        it('sends no limits key without cpuMs, and none on an ordinary create', async () => {
+            const { user, actor } = await makeUser();
+            const path = `/${user.username}/nocpu.worker.js`;
+            await writeSource(actor, user.id, path, 'export default {}');
+            const scriptName = `dt-nocpu-${user.username}`;
+            await deployTo(actor, path, scriptName);
+            expect(uploadedMetadata()).not.toHaveProperty('limits');
+
+            const name = `plain-${user.username}`;
+            await inCtx(actor, () =>
+                target.create({ appId: '', workerName: name, filePath: path }),
+            );
+            const metadata = uploadedMetadata();
+            expect(metadata).not.toHaveProperty('limits');
+            expect(Object.keys(metadata)).toEqual([
+                'body_part',
+                'compatibility_flags',
+                'compatibility_date',
+                'bindings',
+            ]);
+        });
+
         it('refuses a file another user owns, even with manage on it', async () => {
             const owner = await makeUser();
             const deployer = await makeUser();

src/backend/drivers/workers/WorkerDriver.test.ts

@@ -3,6 +3,7 @@
 //  however the interactions with the Puter API are
 import {
     afterAll,
+    afterEach,
     beforeAll,
     beforeEach,
     describe,
@@ -13,6 +14,7 @@ import {
 import { Actor, makeActor as resolveActor } from '../../core/actor.js';
 import { runWithContext } from '../../core/context.js';
 import { PuterServer } from '../../server.js';
+import { DEFAULT_FREE_SUBSCRIPTION } from '../../services/metering/consts.js';
 import {
     setupPuterTestEnv,
     setupTestServer,
@@ -797,6 +799,174 @@ describe('WorkerDriver', () => {
         });
     });
 
+    describe('rented-unit cap', () => {
+        const FREE_CAP = 16;
+        const PAID_CAP = 100;
+
+        // The cap sits after the deploy-credentials gate, so the driver needs
+        // credentials to reach it; a create under the cap then fails on the
+        // missing source file, which is what tells it past the cap check.
+        let originalWorkersConfig: unknown;
+        let originalEvents: unknown;
+        beforeAll(() => {
+            const driverConfig = (
+                target as unknown as { config: Record<string, unknown> }
+            ).config;
+            originalEvents = driverConfig.events;
+            driverConfig.events = { userHandlers: true };
+            originalWorkersConfig = driverConfig.workers;
+            driverConfig.workers = {
+                XAUTHKEY: 'test-key',
+                ACCOUNTID: 'test-account',
+            };
+        });
+        afterAll(() => {
+            const driverConfig = (
+                target as unknown as { config: Record<string, unknown> }
+            ).config;
+            driverConfig.workers = originalWorkersConfig;
+            driverConfig.events = originalEvents;
+        });
+
+        const holding = (
+            plan: string,
+            held: { workers?: number; scopes?: number; owned?: number },
+        ) => {
+            vi.spyOn(
+                server.services.metering,
+                'getActorSubscription',
+            ).mockResolvedValue({ id: plan } as never);
+            vi.spyOn(
+                server.stores.subdomain,
+                'countByUserIdAndPrefix',
+            ).mockResolvedValue(held.workers ?? 0);
+            vi.spyOn(
+                server.stores.eventHandler,
+                'countUserScopesWithHandlersForUser',
+            ).mockResolvedValue(held.scopes ?? 0);
+            vi.spyOn(
+                server.stores.eventHandler,
+                'countAppsWithHandlersForOwner',
+            ).mockResolvedValue(held.owned ?? 0);
+        };
+        afterEach(() => {
+            vi.restoreAllMocks();
+        });
+
+        const createWorker = () =>
+            inCtx(() =>
+                target.create({
+                    appId: 'test-app',
+                    workerName: 'capped',
+                    filePath: '/test.js',
+                }),
+            );
+
+        it('keeps the flat 100-worker cap over ordinary workers with user scopes off', async () => {
+            const driverConfig = (
+                target as unknown as { config: Record<string, unknown> }
+            ).config;
+            driverConfig.events = { userHandlers: false };
+            try {
+                // 16 workers is fine here, and scopes and owned apps don't count.
+                holding(DEFAULT_FREE_SUBSCRIPTION, {
+                    workers: 16,
+                    scopes: 50,
+                    owned: 50,
+                });
+                const list = vi
+                    .spyOn(server.stores.subdomain, 'listByUserIdAndPrefix')
+                    .mockResolvedValue(
+                        Array.from({ length: 16 }, () => ({})) as never,
+                    );
+                await expect(createWorker()).rejects.not.toMatchObject({
+                    statusCode: 403,
+                });
+
+                list.mockResolvedValue(
+                    Array.from({ length: 100 }, () => ({})) as never,
+                );
+                await expect(createWorker()).rejects.toMatchObject({
+                    statusCode: 403,
+                    message: 'Worker limit reached (max 100)',
+                    legacyCode: 'forbidden',
+                });
+            } finally {
+                driverConfig.events = { userHandlers: true };
+            }
+        });
+
+        it('refuses a free user one unit past the cap, across all three sources', async () => {
+            for (const held of [
+                { workers: FREE_CAP },
+                { scopes: FREE_CAP },
+                { owned: FREE_CAP },
+                { workers: 10, scopes: 4, owned: 2 },
+            ]) {
+                holding(DEFAULT_FREE_SUBSCRIPTION, held);
+                await expect(createWorker()).rejects.toMatchObject({
+                    statusCode: 403,
+                    legacyCode: 'forbidden',
+                });
+            }
+        });
+
+        it('lets a free user create up to the cap', async () => {
+            holding(DEFAULT_FREE_SUBSCRIPTION, {
+                workers: 10,
+                scopes: 3,
+                owned: 2,
+            });
+            await expect(createWorker()).rejects.not.toMatchObject({
+                statusCode: 403,
+            });
+        });
+
+        it('refuses a paid user the unit after 100', async () => {
+            holding('paid-plan', { workers: 90, scopes: 5, owned: 5 });
+            await expect(createWorker()).rejects.toMatchObject({
+                statusCode: 403,
+            });
+            holding('paid-plan', { workers: 90, scopes: 5, owned: 4 });
+            await expect(createWorker()).rejects.not.toMatchObject({
+                statusCode: 403,
+            });
+            expect(PAID_CAP).toBe(100);
+        });
+
+        it('keeps a free user over the cap, redeploys what they hold, and refuses a new one', async () => {
+            holding(DEFAULT_FREE_SUBSCRIPTION, { workers: 20 });
+            const user = actor.user;
+            const exists = vi
+                .spyOn(server.stores.subdomain, 'getBySubdomain')
+                .mockImplementation(
+                    async (name: string) =>
+                        (name === 'workers.puter.capped'
+                            ? { user_id: user.id, uuid: 'existing-uuid' }
+                            : null) as never,
+                );
+            const noApp = makeActor({ app: undefined });
+            const create = (workerName: string) =>
+                inCtx(
+                    () =>
+                        target.create({
+                            appId: '',
+                            workerName,
+                            filePath: '/test.js',
+                        }),
+                    noApp,
+                );
+
+            await expect(create('capped')).rejects.not.toMatchObject({
+                statusCode: 403,
+            });
+            await expect(create('another')).rejects.toMatchObject({
+                statusCode: 403,
+            });
+            exists.mockRestore();
+        });
+    });
+
     describe('admission bypass', () => {
         const unverified = (): Actor =>
             makeActor({

src/backend/drivers/workers/WorkerDriver.ts

@@ -43,6 +43,7 @@ import { PuterDriver } from '../types.js';
 import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
 import { awaitLater } from '../../util/concurrency.js';
 import { isUniqueViolation } from '../../util/dbError.js';
+import { assertRentedUnitRoom } from '../../util/rentedUnits.js';
 import { loadFileInput } from '../util/fileInput.js';
 import {
     normalizeLimit,
@@ -57,6 +58,7 @@ const CF_BASE_URL = 'https://api.cloudflare.com/client/v4/accounts';
 export const WORKER_COMPATIBILITY_DATE = '2025-07-15';
 export const WORKER_COMPATIBILITY_FLAGS = ['global_fetch_strictly_public'];
 const WORKER_NAME_REGEX = /^[a-zA-Z0-9_-]+$/;
+/** Cap on ordinary workers while user-scope handlers are off. */
 const MAX_WORKERS_PER_USER = 100;
 
 /**
@@ -179,12 +181,17 @@ export interface InternalDeployTarget {
      * scripts belong to what.
      */
     tags?: string[];
+    /**
+     * Per-invocation CPU limit in milliseconds, sent with the upload. Unset
+     * sends no limit. Ignored by the local runtime.
+     */
+    cpuMs?: number;
 }
 
 const MAX_SOURCE_SIZE = 10 * 1024 * 1024; // 10 MB
 // How far to scan an app's child apps when resolving which workers it may
-// see. A user is capped at MAX_WORKERS_PER_USER workers, so only that many
-// child apps can actually own one; the ceiling is generous slack over that.
+// see. A user is capped at their plan's rented units, so only that many
+// child apps can actually own a worker; the ceiling is generous slack over that.
 const CHILD_APP_SCAN_LIMIT = 1000;
 
 // -- Preamble --------------------------------------------------------
@@ -452,26 +459,34 @@ export class WorkerDriver extends PuterDriver {
 
         this.#requireCfConfig();
 
-        // Quota check — count existing workers.puter.* subdomains owned by user
-        //
-        // A deploy target creates no such row, so this count can neither see
-        // its scripts nor be grown by them; running it would only make an
-        // unrelated worker quota decide whether these deploy. Whatever bounds
-        // them has to be counted where they actually live, and belongs to the
-        // caller that owns their lifecycle.
-        const existingWorkers =
-            skipAdmission || deployTarget
-                ? []
-                : await this.stores.subdomain.listByUserIdAndPrefix(
-                      actor.user.id,
-                      WORKER_SUBDOMAIN_PREFIX,
-                  );
-        if (existingWorkers.length >= MAX_WORKERS_PER_USER) {
-            throw new HttpError(
-                403,
-                `Worker limit reached (max ${MAX_WORKERS_PER_USER})`,
-                { legacyCode: 'forbidden' },
-            );
+        // With user scopes on, a new worker is one more rented unit under the
+        // plan's cap, and redeploying an existing worker is never refused.
+        // Deploy targets create no worker row and are bounded by their caller.
+        if (this.config.events?.userHandlers === true) {
+            if (!skipAdmission && !deployTarget && !existingSub) {
+                await assertRentedUnitRoom(
+                    {
+                        stores: this.stores,
+                        metering: this.services.metering,
+                    },
+                    actor,
+                );
+            }
+        } else {
+            const existingWorkers =
+                skipAdmission || deployTarget
+                    ? []
+                    : await this.stores.subdomain.listByUserIdAndPrefix(
+                          actor.user.id,
+                          WORKER_SUBDOMAIN_PREFIX,
+                      );
+            if (existingWorkers.length >= MAX_WORKERS_PER_USER) {
+                throw new HttpError(
+                    403,
+                    `Worker limit reached (max ${MAX_WORKERS_PER_USER})`,
+                    { legacyCode: 'forbidden' },
+                );
+            }
         }
 
         // Read source file. loadFileInput runs the read-ACL check internally
@@ -940,6 +955,7 @@ export class WorkerDriver extends PuterDriver {
             compatibility_flags: WORKER_COMPATIBILITY_FLAGS,
             compatibility_date: WORKER_COMPATIBILITY_DATE,
             ...(target?.tags ? { tags: target.tags } : {}),
+            ...(target?.cpuMs ? { limits: { cpu_ms: target.cpuMs } } : {}),
             bindings: [
                 // A script deployed without a token gets no binding at all,
                 // rather than an empty one for its code to find.

src/backend/package.json

@@ -57,7 +57,7 @@
     "openai": "^6.34.0",
     "otpauth": "^9.5.2",
     "pg": "^8.21.0",
-    "postal-mime": "^3.0.1",
+    "postal-mime": "^4.2.0",
     "replicate": "^1.0.0",
     "sharp": "^0.35.5",
     "smtp-server": "^3.19.24",

src/backend/server.test.ts

@@ -36,6 +36,10 @@ import { Controller, Get } from './core/http/decorators.ts';
 import { HttpError } from './core/http/HttpError.ts';
 import { extension, extensionStore } from './extensions.ts';
 import { PuterServer } from './server.ts';
+import {
+    JobScheduler,
+    type JobDefinition,
+} from './services/jobs/JobScheduler.ts';
 import {
     allocateEphemeralPort,
     createTestUser,
@@ -1488,6 +1492,104 @@ describe('PuterServer keep-alive timeout', () => {
 });
 
 describe('PuterServer lifecycle hooks', () => {
+    it('fails startup before running later hooks when a start hook rejects', async () => {
+        const failure = new Error('startup failed');
+        const later = vi.fn();
+        class FailingStart extends JobScheduler {
+            override async onServerStart() {
+                throw failure;
+            }
+        }
+        class LaterStart extends JobScheduler {
+            override async onServerStart() {
+                later();
+            }
+        }
+        extension.registerService('failingStart', FailingStart as never);
+        extension.registerService('laterStart', LaterStart as never);
+        try {
+            await expect(setupTestServer()).rejects.toBe(failure);
+            expect(later).not.toHaveBeenCalled();
+        } finally {
+            delete (extensionStore.services as Record<string, unknown>)
+                .failingStart;
+            delete (extensionStore.services as Record<string, unknown>)
+                .laterStart;
+        }
+    });
+
+    it.each([false, true])(
+        'continues teardown after failing hooks with listener callback %s',
+        async (prepareFirst) => {
+            const port = await allocateEphemeralPort();
+            const server = await setupTestServer(
+                { port } as unknown as IConfig,
+                { listen: true },
+            );
+            const logged = vi
+                .spyOn(console, 'error')
+                .mockImplementation(() => {});
+            const ran: string[] = [];
+            class FailingHooks {
+                onServerPrepareShutdown() {
+                    ran.push('prepare:failed');
+                    throw new Error('prepare failed');
+                }
+                async onServerShutdown() {
+                    ran.push('shutdown:failed');
+                    throw new Error('shutdown failed');
+                }
+            }
+            (server.services as Record<string, unknown>).fails =
+                new FailingHooks();
+            (server.services as Record<string, unknown>).after = {
+                onServerPrepareShutdown: () => ran.push('prepare:after'),
+                onServerShutdown: () => ran.push('shutdown:after'),
+            };
+            (server.clients as Record<string, unknown>).last = {
+                onServerShutdown: () => ran.push('shutdown:client'),
+            };
+            try {
+                if (prepareFirst) {
+                    await server.prepareShutdown();
+                    await vi.waitFor(() =>
+                        expect(ran).toContain('prepare:after'),
+                    );
+                }
+                await server.shutdown();
+                expect(ran).toEqual([
+                    'prepare:failed',
+                    'prepare:after',
+                    'shutdown:failed',
+                    'shutdown:after',
+                    'shutdown:client',
+                ]);
+                expect(logged).toHaveBeenCalledWith(
+                    '[server] onServerPrepareShutdown failed in FailingHooks',
+                    expect.any(Error),
+                );
+                expect(logged).toHaveBeenCalledWith(
+                    '[server] onServerShutdown failed in FailingHooks',
+                    expect.any(Error),
+                );
+                await expect(
+                    new Promise<void>((resolve, reject) => {
+                        const socket = net.connect(port, '127.0.0.1');
+                        socket.on('connect', () => {
+                            socket.destroy();
+                            resolve();
+                        });
+                        socket.on('error', reject);
+                    }),
+                ).rejects.toMatchObject({ code: 'ECONNREFUSED' });
+            } finally {
+                delete (server.services as Record<string, unknown>).fails;
+                await server.shutdown();
+                logged.mockRestore();
+            }
+        },
+    );
+
     it('runs drain-start and prepare-shutdown bottom-up, once, before tearing down', async () => {
         const port = await allocateEphemeralPort();
         const server = await setupTestServer({ port } as unknown as IConfig, {
@@ -1537,6 +1639,136 @@ describe('PuterServer lifecycle hooks', () => {
             'shutdown:client',
         ]);
     });
+
+    it('keeps accepting connections until every drain-start hook settles, without one holding up another', async () => {
+        const port = await allocateEphemeralPort();
+        const server = await setupTestServer({ port } as unknown as IConfig, {
+            listen: true,
+        });
+        let release!: () => void;
+        const held = new Promise<void>((resolve) => (release = resolve));
+        let otherRan = false;
+        (server.services as unknown as Record<string, unknown>).holdsListener =
+            { onServerDrainStart: () => held };
+        (server.drivers as unknown as Record<string, unknown>).other = {
+            onServerDrainStart: () => {
+                otherRan = true;
+            },
+        };
+
+        const connect = () =>
+            new Promise<void>((resolve, reject) => {
+                const socket = net.connect(port, '127.0.0.1');
+                socket.on('connect', () => {
+                    socket.destroy();
+                    resolve();
+                });
+                socket.on('error', reject);
+            });
+
+        const prepared = server.prepareShutdown({ drainMs: 1_000 });
+        await new Promise((resolve) => setTimeout(resolve, 20));
+        expect(otherRan).toBe(true);
+        await expect(connect()).resolves.toBeUndefined();
+
+        release();
+        await prepared;
+        await expect(connect()).rejects.toMatchObject({
+            code: 'ECONNREFUSED',
+        });
+        await server.shutdown();
+    });
+
+    it('runs every drain-start hook when one of them throws', async () => {
+        const port = await allocateEphemeralPort();
+        const server = await setupTestServer({ port } as unknown as IConfig, {
+            listen: true,
+        });
+        vi.spyOn(console, 'error').mockImplementation(() => {});
+        const ran: string[] = [];
+        (server.stores as unknown as Record<string, unknown>).fails = {
+            onServerDrainStart: () => {
+                throw new Error('drain hook failed');
+            },
+        };
+        (server.services as unknown as Record<string, unknown>).after = {
+            onServerDrainStart: () => {
+                ran.push('after');
+            },
+        };
+
+        await server.prepareShutdown({ drainMs: 1_000 });
+        await server.shutdown();
+
+        expect(ran).toEqual(['after']);
+        expect(console.error).toHaveBeenCalledWith(
+            expect.stringContaining('onServerDrainStart failed'),
+            expect.any(Error),
+        );
+    });
+});
+
+describe('PuterServer shutdown during prepare', () => {
+    it('waits for prepare hooks the drain already started before tearing down', async () => {
+        const port = await allocateEphemeralPort();
+        const server = await setupTestServer({ port } as unknown as IConfig, {
+            listen: true,
+        });
+        const order: string[] = [];
+        (server.services as unknown as Record<string, unknown>).slowPrepare = {
+            onServerPrepareShutdown: async () => {
+                order.push('prepare:start');
+                await new Promise((resolve) => setTimeout(resolve, 200));
+                order.push('prepare:done');
+            },
+            onServerShutdown: () => {
+                order.push('shutdown');
+            },
+        };
+
+        await server.prepareShutdown({ drainMs: 0 });
+        // The listener's close callback starts the prepare hooks.
+        await new Promise((resolve) => setTimeout(resolve, 50));
+        await server.shutdown();
+
+        expect(order).toEqual(['prepare:start', 'prepare:done', 'shutdown']);
+    });
+});
+
+/**
+ * An extension may take over `services.jobs` (a fleet-wide scheduler). Core
+ * services register their jobs on it at start, so it must keep the built-in's
+ * first place in the layer.
+ */
+describe('PuterServer jobs replacement', () => {
+    it('lets an extension service registered as `jobs` replace the built-in and keep its first position', async () => {
+        const registered: string[] = [];
+        class ReplacementJobs extends JobScheduler {
+            override register(name: string, definition: JobDefinition) {
+                registered.push(name);
+                super.register(name, definition);
+            }
+        }
+        const started = vi.spyOn(JobScheduler.prototype, 'onServerStart');
+        extension.registerService('jobs', ReplacementJobs as never);
+        let server: PuterServer | undefined;
+        try {
+            server = await setupTestServer();
+            const { jobs } = server.services;
+            expect(jobs).toBeInstanceOf(ReplacementJobs);
+            expect(Object.keys(server.services)[0]).toBe('jobs');
+            expect(extension.import('service').jobs).toBeInstanceOf(
+                ReplacementJobs,
+            );
+            // Only the replacement gets lifecycle hooks.
+            expect(started.mock.contexts).toEqual([jobs]);
+            expect(registered).toContain('auth.sessions.purge');
+        } finally {
+            delete (extensionStore.services as Record<string, unknown>).jobs;
+            started.mockRestore();
+            await server?.shutdown();
+        }
+    });
 });
 
 /**

src/backend/server.ts

@@ -1743,12 +1743,7 @@ export class PuterServer {
      */
     async #runHook(
         order: 'bottomUp' | 'topDown',
-        hook:
-            | 'onServerStart'
-            | 'onServerDrainStart'
-            | 'onServerPrepareShutdown'
-            | 'onServerShutdown',
-        info?: ServerDrainInfo,
+        hook: 'onServerStart' | 'onServerPrepareShutdown' | 'onServerShutdown',
     ) {
         const layers = [
             this.clients,
@@ -1760,35 +1755,77 @@ export class PuterServer {
         if (order === 'topDown') layers.reverse();
         for (const layer of layers) {
             for (const instance of Object.values(layer) as WithLifecycle[]) {
-                await instance[hook]?.(info);
+                try {
+                    await instance[hook]?.();
+                } catch (err) {
+                    if (hook === 'onServerStart') throw err;
+                    console.error(
+                        `[server] ${hook} failed in ${instance.constructor.name}`,
+                        err,
+                    );
+                }
             }
         }
     }
 
-    #prepareShutdownHooksRan = false;
+    /**
+     * Run every `onServerDrainStart` concurrently, so one hook holding the
+     * listener open doesn't delay the others, and one failing doesn't skip the
+     * rest.
+     */
+    async #runDrainStartHooks(info: ServerDrainInfo) {
+        const instances = [
+            this.clients,
+            this.stores,
+            this.services,
+            this.controllers,
+            this.drivers,
+        ].flatMap((layer) => Object.values(layer) as WithLifecycle[]);
+        const results = await Promise.allSettled(
+            instances.map(async (instance) =>
+                instance.onServerDrainStart?.(info),
+            ),
+        );
+        results.forEach((result, i) => {
+            if (result.status === 'rejected') {
+                console.error(
+                    `[server] onServerDrainStart failed in ${instances[i].constructor.name}`,
+                    result.reason,
+                );
+            }
+        });
+    }
+
+    #prepareShutdownHooks: Promise<void> | null = null;
 
     /**
      * Run every layer's `onServerPrepareShutdown` exactly once, whichever of
-     * `prepareShutdown()` / `shutdown()` gets there first.
+     * `prepareShutdown()` / `shutdown()` gets there first; the other waits for
+     * that run to finish.
      */
-    async #runPrepareShutdownHooks() {
-        if (this.#prepareShutdownHooksRan) return;
-        this.#prepareShutdownHooksRan = true;
-        await this.#runHook('bottomUp', 'onServerPrepareShutdown');
+    #runPrepareShutdownHooks(): Promise<void> {
+        this.#prepareShutdownHooks ??= this.#runHook(
+            'bottomUp',
+            'onServerPrepareShutdown',
+        );
+        return this.#prepareShutdownHooks;
     }
 
     async prepareShutdown(info: ServerDrainInfo = {}) {
         if (this.#server) {
-            try {
-                await this.#runHook('bottomUp', 'onServerDrainStart', info);
-            } catch (err) {
-                console.error('[server] onServerDrainStart failed', err);
-            }
+            await this.#runDrainStartHooks(info);
             this.#server.close(async () => {
                 console.log(
                     'PuterServer has stopped accepting new connections',
                 );
-                await this.#runPrepareShutdownHooks();
+                try {
+                    await this.#runPrepareShutdownHooks();
+                } catch (err) {
+                    console.error(
+                        '[server] prepare-shutdown hooks failed',
+                        err,
+                    );
+                }
             });
         }
     }

src/backend/services/auth/AuthService.test.ts

@@ -24,6 +24,7 @@ import { makeActor, type Actor } from '../../core/actor.js';
 import { PuterServer } from '../../server.js';
 import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js';
 import { setupTestServer } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import { generateDefaultFsentries } from '../../util/userProvisioning.js';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS } from '../permission/consts.js';
@@ -4048,7 +4049,6 @@ describe('AuthService (integration)', () => {
     });
 
     describe('purgeEndedSessions', () => {
-        const GATE_KEY = 'sessions:purge:gate';
         const DAY = 24 * 60 * 60;
 
         const makeSession = async (
@@ -4077,8 +4077,15 @@ describe('AuthService (integration)', () => {
             return rows.length > 0;
         };
 
+        const idOf = async (uuid: string) => {
+            const [row] = await server.clients.db.read(
+                'SELECT `id` FROM `sessions` WHERE `uuid` = ?',
+                [uuid],
+            );
+            return Number(row.id);
+        };
+
         it('deletes rows ended past their retention and keeps the rest', async () => {
-            await server.clients.redis.del(GATE_KEY);
             const user = await makeUser();
             const id = user.id;
             const oldRevoked = await makeSession(id, 'asset', 'revoked_at', 31);
@@ -4100,7 +4107,7 @@ describe('AuthService (integration)', () => {
             const live = await makeSession(id, 'web');
 
             expect(
-                await authService.purgeEndedSessions(),
+                await authService.purgeEndedSessions(testJobContext()),
             ).toBeGreaterThanOrEqual(3);
 
             expect(await exists(oldRevoked)).toBe(false);
@@ -4112,34 +4119,84 @@ describe('AuthService (integration)', () => {
             expect(await exists(live)).toBe(true);
         });
 
-        it('skips while another run holds the gate', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            await server.clients.redis.set(GATE_KEY, 'other-node');
+        it('starts from the saved cursor and clears it once the walk is done', async () => {
             const user = await makeUser();
-            const oldRevoked = await makeSession(
-                user.id,
-                'asset',
-                'revoked_at',
-                31,
-            );
+            const below = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const above = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const ctx = testJobContext({ afterId: await idOf(below) });
 
-            expect(await authService.purgeEndedSessions()).toBeNull();
-            expect(await exists(oldRevoked)).toBe(true);
-            await server.clients.redis.del(GATE_KEY);
+            expect(
+                await authService.purgeEndedSessions(ctx),
+            ).toBeGreaterThanOrEqual(1);
+            expect(await exists(below)).toBe(true);
+            expect(await exists(above)).toBe(false);
+            expect(ctx.saved).toEqual([null]);
         });
 
-        it('frees the gate when the walk fails', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            const spy = vi
-                .spyOn(server.clients.db, 'read')
-                .mockRejectedValueOnce(new Error('db down'));
+        it('clears its own checkpoints once a walk completes, so the next run starts at zero', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 11,
+            );
+            const ranges: number[] = [];
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    return 0;
+                });
             try {
-                await expect(authService.purgeEndedSessions()).rejects.toThrow(
-                    'db down',
+                const first = testJobContext();
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(11);
+                // Checkpointed at ten ranges, then cleared at the end.
+                expect(first.saved).toEqual([{ afterId: SPAN * 10 }, null]);
+
+                ranges.length = 0;
+                await authService.purgeEndedSessions(
+                    testJobContext(first.saved.at(-1)),
                 );
-                expect(await server.clients.redis.get(GATE_KEY)).toBeNull();
+                expect(ranges[0]).toBe(0);
+                expect(ranges).toHaveLength(11);
             } finally {
-                spy.mockRestore();
+                purge.mockRestore();
+                vi.restoreAllMocks();
+            }
+        });
+
+        it('stops between ranges once aborted, and the next run resumes at the saved cursor', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 30,
+            );
+            const ranges: number[] = [];
+            const first = testJobContext();
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    // Aborted mid-walk, as a lost lease or a shutdown would.
+                    if (ranges.length === 12) first.abort();
+                    return 0;
+                });
+            try {
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(12);
+                // Saved every 10 ranges, and where it stopped.
+                expect(first.saved).toEqual([
+                    { afterId: SPAN * 10 },
+                    { afterId: SPAN * 12 },
+                ]);
+
+                ranges.length = 0;
+                const next = testJobContext(first.saved.at(-1));
+                await authService.purgeEndedSessions(next);
+                expect(ranges[0]).toBe(SPAN * 12);
+                expect(ranges.at(-1)).toBe(SPAN * 29);
+                expect(next.saved.at(-1)).toBeNull();
+            } finally {
+                purge.mockRestore();
+                vi.restoreAllMocks();
             }
         });
     });

src/backend/services/auth/AuthService.ts

@@ -46,6 +46,7 @@ import type { puterServices } from '../index';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS, PERMISSION_MAX_LEN } from '../permission/consts';
 import type { GrantMeta } from '../permission/PermissionService.js';
+import { savedAfterId, type JobRunContext } from '../jobs/JobScheduler.js';
 import { PuterService } from '../types';
 import type {
     AccessTokenPayload,
@@ -64,13 +65,14 @@ const MAX_ORIGIN_UID_GENERATIONS = 8;
 // rows double as sign-in history, so they stay longer.
 const SESSION_RETENTION_SECONDS = 30 * 24 * 60 * 60;
 const WEB_SESSION_RETENTION_SECONDS = 365 * 24 * 60 * 60;
-// Every node checks hourly; the gate lets one node through per week.
-const SESSION_PURGE_CHECK_MS = 60 * 60 * 1000;
-const SESSION_PURGE_GATE_KEY = 'sessions:purge:gate';
-const SESSION_PURGE_GATE_SECONDS = 7 * 24 * 60 * 60;
+const SESSION_PURGE_JOB = 'auth.sessions.purge';
+const SESSION_PURGE_EVERY_MS = 7 * 24 * 60 * 60 * 1000;
+const SESSION_PURGE_LEASE_MS = 30 * 60 * 1000;
 // Ids per DELETE, with a pause between, so replicas keep up.
 const SESSION_PURGE_SPAN = 5000;
 const SESSION_PURGE_PAUSE_MS = 50;
+/** Ranges walked between saves of the cursor. */
+const SESSION_PURGE_SAVE_EVERY = 10;
 
 const nowSeconds = (): number => Math.floor(Date.now() / 1000);
 
@@ -138,11 +140,17 @@ export interface AuthResult {
 export class AuthService extends PuterService {
     declare protected services: LayerInstances<typeof puterServices>;
 
-    #sessionPurge: ReturnType<typeof setInterval> | null = null;
-    #stopping = false;
-
     override onServerStart(): void {
-        this.#armSessionPurge();
+        this.services.jobs.register(SESSION_PURGE_JOB, {
+            every: SESSION_PURGE_EVERY_MS,
+            scope: 'fleet',
+            nearPrimary: true,
+            leaseMs: SESSION_PURGE_LEASE_MS,
+            run: async (ctx) => {
+                const purged = await this.purgeEndedSessions(ctx);
+                console.log(`[auth] purged ${purged} ended sessions`);
+            },
+        });
 
         // Users implicitly hold read access to their own email — needed for
         // any permission-gated path that asks for `user:<uuid>:email:read`
@@ -164,85 +172,54 @@ export class AuthService extends PuterService {
         });
     }
 
-    override onServerPrepareShutdown(): void {
-        this.#stopping = true;
-        if (this.#sessionPurge) clearInterval(this.#sessionPurge);
-        this.#sessionPurge = null;
-    }
-
     // -- Session purge -----------------------------------------------
 
     /**
      * Delete sessions revoked or expired longer ago than their retention,
-     * walking the table by id. Returns the rows deleted, or null when another
-     * node already holds this week's gate.
+     * walking the table by id from the cursor an interrupted run saved. Returns
+     * the rows deleted; stops early once `signal` aborts.
      */
-    async purgeEndedSessions(): Promise<number | null> {
-        const claimed = await this.clients.redis.set(
-            SESSION_PURGE_GATE_KEY,
-            String(Date.now()),
-            'EX',
-            SESSION_PURGE_GATE_SECONDS,
-            'NX',
-        );
-        if (claimed !== 'OK') return null;
-
+    async purgeEndedSessions({
+        signal,
+        state,
+        saveState,
+    }: JobRunContext): Promise<number> {
         const now = nowSeconds();
         const cutoffs = {
             cutoff: now - SESSION_RETENTION_SECONDS,
             webCutoff: now - WEB_SESSION_RETENTION_SECONDS,
         };
+        const maxId = await this.stores.session.maxId();
+        let afterId = savedAfterId(state);
         let purged = 0;
-        let finished = false;
-        try {
-            const maxId = await this.stores.session.maxId();
-            for (
-                let afterId = 0;
-                afterId < maxId;
-                afterId += SESSION_PURGE_SPAN
-            ) {
-                if (this.#stopping) return purged;
-                purged += await this.stores.session.purgeEndedInRange(
-                    afterId,
-                    SESSION_PURGE_SPAN,
-                    cutoffs,
-                );
-                await new Promise((resolve) =>
-                    setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
-                );
-            }
-            finished = true;
-            return purged;
-        } finally {
-            // A walk cut short frees the gate, so another node restarts it on
-            // its next check instead of next week.
-            if (!finished) {
+        let unsaved = 0;
+        while (afterId < maxId) {
+            if (signal.aborted) {
+                // Kept for the next run if this one still holds the job.
                 try {
-                    await this.clients.redis.del(SESSION_PURGE_GATE_KEY);
+                    await saveState({ afterId });
                 } catch {
-                    // The gate expires on its own.
+                    // Lost the job; the next run redoes these ranges.
                 }
+                return purged;
             }
-        }
-    }
-
-    #armSessionPurge(): void {
-        const timer = setInterval(() => {
-            void this.#runSessionPurge();
-        }, SESSION_PURGE_CHECK_MS);
-        timer.unref?.();
-        this.#sessionPurge = timer;
-    }
-
-    async #runSessionPurge(): Promise<void> {
-        try {
-            const purged = await this.purgeEndedSessions();
-            if (purged !== null) {
-                console.log(`[auth] purged ${purged} ended sessions`);
+            purged += await this.stores.session.purgeEndedInRange(
+                afterId,
+                SESSION_PURGE_SPAN,
+                cutoffs,
+            );
+            afterId += SESSION_PURGE_SPAN;
+            if (++unsaved >= SESSION_PURGE_SAVE_EVERY) {
+                await saveState({ afterId });
+                unsaved = 0;
             }
-        } catch (err) {
-            console.warn('[auth] session purge failed', err);
+            await new Promise((resolve) =>
+                setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
+            );
         }
+        // Clears any checkpoint, this run's included, so next week starts at 0.
+        await saveState(null);
+        return purged;
     }
 
     // -- Public API --------------------------------------------------

src/backend/services/broadcast/BroadcastService.test.ts

@@ -1149,3 +1149,154 @@ describe('BroadcastService — same-cluster pub/sub fan-out', () => {
         }
     });
 });
+
+describe('BroadcastService shutdown drain', () => {
+    let server: PuterServer;
+    let broadcast: BroadcastService;
+
+    beforeEach(async () => {
+        axiosRequestMock.mockReset();
+        axiosRequestMock.mockResolvedValue({ status: 200, data: 'ok' });
+        server = await setupTestServer({
+            broadcast: {
+                webhook: { peerId: SELF_PEER_ID, secret: SELF_SECRET },
+                peers: [
+                    {
+                        peerId: PEER_ID,
+                        webhook: true,
+                        webhook_url: PEER_URL,
+                        webhook_secret: PEER_SECRET,
+                    },
+                ],
+                outbound_flush_ms: 10,
+            },
+        } as never);
+        broadcast = server.services.broadcast as unknown as BroadcastService;
+    });
+
+    afterEach(async () => {
+        vi.useRealTimers();
+        await broadcast.onServerPrepareShutdown();
+        await broadcast.onServerShutdown();
+        await server.shutdown();
+    });
+
+    const emit = (hash: string) =>
+        server.clients.event.emit(
+            'outer.fs.write-hash' as never,
+            { hash, uuid: hash } as never,
+            {},
+        );
+
+    it.each(['onServerPrepareShutdown', 'onServerShutdown'] as const)(
+        'waits for the active flush and sends its tail in %s',
+        async (hook) => {
+            let release!: () => void;
+            const held = new Promise<void>((resolve) => {
+                release = resolve;
+            });
+            let releaseTail!: () => void;
+            const tailHeld = new Promise<void>((resolve) => {
+                releaseTail = resolve;
+            });
+            axiosRequestMock
+                .mockImplementationOnce(async () => {
+                    await held;
+                    return { status: 200, data: 'ok' };
+                })
+                .mockImplementationOnce(async () => {
+                    await tailHeld;
+                    return { status: 200, data: 'ok' };
+                });
+            emit('first');
+            await vi.waitFor(() =>
+                expect(axiosRequestMock).toHaveBeenCalledTimes(1),
+            );
+            emit('tail');
+            let finished = false;
+            const draining = broadcast[hook]().then(() => {
+                finished = true;
+            });
+            try {
+                await new Promise((resolve) => setTimeout(resolve, 20));
+                expect(finished).toBe(false);
+                release();
+                await vi.waitFor(() =>
+                    expect(axiosRequestMock).toHaveBeenCalledTimes(2),
+                );
+                expect(finished).toBe(false);
+            } finally {
+                release();
+                releaseTail();
+                await draining;
+            }
+            expect(axiosRequestMock).toHaveBeenCalledTimes(2);
+            expect(axiosRequestMock.mock.calls[1][0].data).toContain('tail');
+        },
+    );
+
+    it('keeps normal timer flushing after prepare on an idle server', async () => {
+        await broadcast.onServerPrepareShutdown();
+        vi.useFakeTimers();
+        axiosRequestMock.mockClear();
+        emit('during-drain');
+        await vi.advanceTimersByTimeAsync(10);
+        expect(axiosRequestMock).toHaveBeenCalledTimes(1);
+        expect(axiosRequestMock.mock.calls[0][0].data).toContain(
+            'during-drain',
+        );
+    });
+
+    it('sends events queued after prepare during shutdown and clears the timer', async () => {
+        await broadcast.onServerPrepareShutdown();
+        vi.useFakeTimers();
+        axiosRequestMock.mockClear();
+        emit('after-prepare');
+        await broadcast.onServerShutdown();
+        expect(axiosRequestMock).toHaveBeenCalledTimes(1);
+        expect(axiosRequestMock.mock.calls[0][0].data).toContain(
+            'after-prepare',
+        );
+        axiosRequestMock.mockClear();
+        await vi.advanceTimersByTimeAsync(100);
+        expect(axiosRequestMock).not.toHaveBeenCalled();
+    });
+
+    it.each(['onServerPrepareShutdown', 'onServerShutdown'] as const)(
+        'bounds a hung flush in %s',
+        async (hook) => {
+            let release!: () => void;
+            const held = new Promise<void>((resolve) => {
+                release = resolve;
+            });
+            axiosRequestMock.mockImplementationOnce(async () => {
+                await held;
+                return { status: 200, data: 'ok' };
+            });
+            emit('hung');
+            await vi.waitFor(() =>
+                expect(axiosRequestMock).toHaveBeenCalledTimes(1),
+            );
+            vi.useFakeTimers();
+            emit('tail');
+            let finished = false;
+            const draining = broadcast[hook]().then(() => {
+                finished = true;
+            });
+            try {
+                await vi.advanceTimersByTimeAsync(4_999);
+                expect(finished).toBe(false);
+                await vi.advanceTimersByTimeAsync(1);
+                await draining;
+                release();
+                await vi.advanceTimersByTimeAsync(100);
+                expect(axiosRequestMock).toHaveBeenCalledTimes(
+                    hook === 'onServerPrepareShutdown' ? 2 : 1,
+                );
+            } finally {
+                release();
+                vi.useRealTimers();
+            }
+        },
+    );
+});

src/backend/services/broadcast/BroadcastService.ts

@@ -21,6 +21,8 @@ import axios from 'axios';
 import { createHmac, randomUUID, timingSafeEqual } from 'node:crypto';
 import { Agent as HttpsAgent } from 'node:https';
 import { IBroadcastPeerConfig } from '../../types.js';
+import { settleWithin } from '../../util/concurrency.js';
+import { FINAL_FLUSH_TIMEOUT_MS } from '../../util/gracefulShutdown.js';
 import { PuterService } from '../types.js';
 
 // -- Wire types ------------------------------------------------------
@@ -121,7 +123,8 @@ export class BroadcastService extends PuterService {
     /** Per peer, serialized events it has not accepted yet, oldest first. */
     #outboundBacklogs = new Map<IBroadcastPeerConfig, string[]>();
     #outboundFlushTimer: ReturnType<typeof setTimeout> | null = null;
-    #outboundIsFlushing = false;
+    #outboundFlush: Promise<void> | null = null;
+    #outboundDraining = false;
 
     #webhookReplayWindowSeconds = 300;
     #outboundFlushMs = 2000;
@@ -165,21 +168,40 @@ export class BroadcastService extends PuterService {
     }
 
     override async onServerPrepareShutdown(): Promise<void> {
+        await this.#drainOutboundEvents();
+        if (this.#redisSub) {
+            try {
+                await this.#redisSub.unsubscribe('pubsub');
+                this.#redisSub.quit();
+            } catch (err) {
+                console.warn('[broadcast] unsubscribe failed', err);
+            } finally {
+                this.#redisSub = null;
+            }
+        }
+    }
+
+    override async onServerShutdown(): Promise<void> {
+        this.#outboundDraining = true;
+        await this.#drainOutboundEvents();
+    }
+
+    async #drainOutboundEvents(): Promise<void> {
         if (this.#outboundFlushTimer) {
             clearTimeout(this.#outboundFlushTimer);
             this.#outboundFlushTimer = null;
         }
-        // Best-effort drain — try one final flush so events queued near
-        // shutdown make it out.
-        try {
-            await this.#flushOutboundEvents();
-        } catch (err) {
-            console.warn('[broadcast] final flush failed', err);
-        }
-        if (this.#redisSub) {
-            await this.#redisSub.unsubscribe('pubsub');
-            this.#redisSub.quit();
-            this.#redisSub = null;
+        const deadline = Date.now() + FINAL_FLUSH_TIMEOUT_MS;
+        const drain = async () => {
+            try {
+                await this.#outboundFlush;
+                if (Date.now() < deadline) await this.#flushOutboundEvents();
+            } catch (err) {
+                console.warn('[broadcast] final flush failed', err);
+            }
+        };
+        if (!(await settleWithin(drain(), FINAL_FLUSH_TIMEOUT_MS))) {
+            console.warn('[broadcast] final flush timed out');
         }
     }
 
@@ -453,24 +475,27 @@ export class BroadcastService extends PuterService {
     }
 
     #scheduleOutboundFlush(): void {
-        if (this.#outboundFlushTimer) return;
-        this.#outboundFlushTimer = setTimeout(() => {
+        if (this.#outboundDraining || this.#outboundFlushTimer) return;
+        this.#outboundFlushTimer = setTimeout(async () => {
             this.#outboundFlushTimer = null;
-            void this.#flushOutboundEvents().catch((err) => {
+            try {
+                await this.#flushOutboundEvents();
+            } catch (err) {
                 console.warn('[broadcast] outbound flush failed', err);
-            });
+            }
         }, this.#outboundFlushMs);
     }
 
-    async #flushOutboundEvents(): Promise<void> {
-        if (
-            this.#outboundIsFlushing ||
-            (this.#outboundQueue.size === 0 &&
-                this.#outboundBacklogs.size === 0)
-        )
-            return;
+    #flushOutboundEvents(): Promise<void> {
+        if (this.#outboundFlush) return this.#outboundFlush;
+        if (this.#outboundQueue.size === 0 && this.#outboundBacklogs.size === 0)
+            return Promise.resolve();
 
-        this.#outboundIsFlushing = true;
+        this.#outboundFlush = this.#sendOutboundEvents();
+        return this.#outboundFlush;
+    }
+
+    async #sendOutboundEvents(): Promise<void> {
         try {
             const fresh = this.#outboundQueue;
             this.#outboundQueue = new Set();
@@ -504,7 +529,7 @@ export class BroadcastService extends PuterService {
             for (const result of flushed)
                 if (result.status === 'rejected') throw result.reason;
         } finally {
-            this.#outboundIsFlushing = false;
+            this.#outboundFlush = null;
             // Anything that arrived during flush, or is still owed to a peer,
             // gets the next tick.
             if (

src/backend/services/events/EventsService.test.ts

@@ -185,7 +185,11 @@ const aclService = () => ({
         !denied.has(resource.path) && !deniedHolders.has(actor.user?.id ?? -1),
     getSafeAclError: async (_actor: Actor, resource: { path: string }) =>
         denied.get(resource.path) === 'forbidden'
-            ? { status: 403, message: 'Forbidden', fields: { code: 'forbidden' } }
+            ? {
+                  status: 403,
+                  message: 'Forbidden',
+                  fields: { code: 'forbidden' },
+              }
             : {
                   status: 404,
                   message: 'Subject does not exist',
@@ -281,8 +285,8 @@ interface ForwardedCall {
 }
 
 /**
- * Each service gets its own outbox. A delivery still in flight when a test
- * ends must land in that test's record, not in the next one's.
+ * Each service gets its own outbox. A delivery still in flight when a test ends
+ * must land in that test's record, not in the next one's.
  */
 const buildService = (
     config: IConfig,
@@ -293,12 +297,14 @@ const buildService = (
     metered: MeteredLine[];
     forwarded: ForwardedCall[];
     eventBus: { on: ReturnType<typeof vi.fn>; emit: ReturnType<typeof vi.fn> };
+    jobs: { register: ReturnType<typeof vi.fn> };
 } => {
     const outbox: Array<{ socket?: string; envelope: DeliveryEnvelope }> = [];
     const counted: DeliveryEnvelope[] = [];
     const lines: MeteredLine[] = [];
     const forwardedCalls: ForwardedCall[] = [];
     const bus = { on: vi.fn(), emit: vi.fn() };
+    const jobs = { register: vi.fn() };
     const built = new EventsService(
         config,
         {
@@ -315,6 +321,7 @@ const buildService = (
             kvShareHandle: kvShareHandleStore,
         } as never,
         {
+            jobs,
             eventForward: {
                 // A deployment with no peers has nowhere to forward to, which
                 // is what every test here is.
@@ -347,10 +354,7 @@ const buildService = (
             acl: aclService(),
             permission: permissionService(),
             metering: {
-                bufferIncrementUsages: (
-                    actor: Actor,
-                    usages: UsageInput[],
-                ) => {
+                bufferIncrementUsages: (actor: Actor, usages: UsageInput[]) => {
                     for (const usage of usages)
                         lines.push({
                             userUuid: actor.user?.uuid,
@@ -376,6 +380,7 @@ const buildService = (
         metered: lines,
         forwarded: forwardedCalls,
         eventBus: bus,
+        jobs,
     };
 };
 
@@ -564,6 +569,59 @@ describe('the feature switch', () => {
         );
     });
 
+    it('hands its table sweeps to the job scheduler and arms only the pending sweep itself', () => {
+        const intervals = vi.spyOn(globalThis, 'setInterval');
+        const timeouts = vi.spyOn(globalThis, 'setTimeout');
+        let jobs: { register: ReturnType<typeof vi.fn> };
+        try {
+            ({ jobs } = buildService({ events: { enabled: true } } as IConfig));
+            // The pending sweep is the region's queue, so every node runs it.
+            expect(intervals.mock.calls.map(([, ms]) => ms)).toEqual([10_000]);
+            expect(timeouts).not.toHaveBeenCalled();
+        } finally {
+            intervals.mockRestore();
+            timeouts.mockRestore();
+        }
+        expect(
+            jobs.register.mock.calls.map(([name, job]) => ({
+                name,
+                every: job.every,
+                scope: job.scope,
+                nearPrimary: job.nearPrimary ?? false,
+                leaseMs: job.leaseMs,
+            })),
+        ).toEqual([
+            {
+                name: 'events.subscriptions.expirySweep',
+                every: 60 * 60_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: 15 * 60_000,
+            },
+            {
+                name: 'events.userHandlers.gc',
+                every: 60 * 60_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: 15 * 60_000,
+            },
+            {
+                name: 'events.subscriptions.creditSweep',
+                every: 15 * 60_000,
+                scope: 'fleet',
+                nearPrimary: false,
+                leaseMs: 10 * 60_000,
+            },
+        ]);
+    });
+
+    it('registers no job when off', () => {
+        const { jobs } = buildService({
+            events: { enabled: false },
+        } as IConfig);
+        expect(jobs.register).not.toHaveBeenCalled();
+    });
+
     it('spends nothing at all on a dispatch when off', async () => {
         const { service: off } = buildService({
             events: { enabled: false },
@@ -1206,7 +1264,9 @@ describe('matching', () => {
     it('stops delivering to a durable row the moment it expires', async () => {
         const { documents, file } = seedTree();
         const now = Math.floor(Date.now() / 1000);
-        const row = (over: Partial<DurableSubscription>): DurableSubscription => ({
+        const row = (
+            over: Partial<DurableSubscription>,
+        ): DurableSubscription => ({
             subId: `durable-${seq}-${over.expiresAt}`,
             token: `f#${documents.uid}`,
             ownerUserId: userId,
@@ -1355,9 +1415,9 @@ describe('matching', () => {
         await dispatch(closed);
         await flush();
 
-        expect(sent.map((s) => (s.envelope.event as { uid: string }).uid)).toEqual([
-            reachable.uid,
-        ]);
+        expect(
+            sent.map((s) => (s.envelope.event as { uid: string }).uid),
+        ).toEqual([reachable.uid]);
     });
 
     it('publishes nothing for an event with no registry entry', async () => {
@@ -1399,9 +1459,9 @@ describe('matching', () => {
         await flush();
 
         expect(sent).toHaveLength(1);
-        expect(
-            (sent[0].envelope.event as { path: string }).path,
-        ).toBe(inside.path);
+        expect((sent[0].envelope.event as { path: string }).path).toBe(
+            inside.path,
+        );
     });
 
     it('does not reach a sibling that only shares the missing name as a prefix', async () => {
@@ -1473,8 +1533,9 @@ describe('coalescing', () => {
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
         expect(sent).toHaveLength(2);
-        expect(sent.map((s) => (s.envelope.event as { uid: string }).uid).sort())
-            .toEqual([file.uid, second.uid].sort());
+        expect(
+            sent.map((s) => (s.envelope.event as { uid: string }).uid).sort(),
+        ).toEqual([file.uid, second.uid].sort());
     });
 
     it('counts exactly what it delivered', async () => {
@@ -1620,9 +1681,9 @@ describe('limits', () => {
             }
         }
 
-        await expect(subscribe(`fs:${documents.uid}`)).resolves.toMatchObject(
-            { subId: expect.any(String) },
-        );
+        await expect(subscribe(`fs:${documents.uid}`)).resolves.toMatchObject({
+            subId: expect.any(String),
+        });
     });
 
     it('refuses unsubscribes past their own per-minute budget', async () => {
@@ -1691,9 +1752,7 @@ describe('limits', () => {
             ),
         );
 
-        expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(
-            1,
-        );
+        expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
         expect(results.filter((r) => r.status === 'rejected')).toHaveLength(
             attempts - 1,
         );
@@ -1890,9 +1949,9 @@ describe('limits', () => {
         ).toHaveLength(EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT);
         // 200 evaluated, 20 never reached: everything past the delivery cap is
         // told it missed something, itself capped.
-        expect(
-            sent.filter((s) => s.envelope.event.op === 'gap'),
-        ).toHaveLength(EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT);
+        expect(sent.filter((s) => s.envelope.event.op === 'gap')).toHaveLength(
+            EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT,
+        );
         expect(
             sent.find((s) => s.envelope.event.op === 'gap')?.envelope.event,
         ).toMatchObject({ reason: 'filter_evaluation_limit' });
@@ -1943,8 +2002,7 @@ describe('the socket surface', () => {
             fire: (event, ...args) =>
                 (
                     handlers.get(event) as
-                        | ((...a: unknown[]) => void)
-                        | undefined
+                        ((...a: unknown[]) => void) | undefined
                 )?.(...args),
         };
     };
@@ -2125,10 +2183,14 @@ it('names the delivery channel the clients listen on', () => {
 
 describe('asking a kv subscription for the value', () => {
     it('is recorded on the row and reported in the view', async () => {
-        const { sub } = await service.subscribe(appActorFor(OWN_APP), socketId, {
-            subject: `kv:${OWN_APP}:cart`,
-            includeValue: true,
-        });
+        const { sub } = await service.subscribe(
+            appActorFor(OWN_APP),
+            socketId,
+            {
+                subject: `kv:${OWN_APP}:cart`,
+                includeValue: true,
+            },
+        );
         expect(sub.includeValue).toBe(true);
 
         const plain = await subscribeKv(`kv:${OWN_APP}:cart`);
@@ -2211,10 +2273,7 @@ describe('resolving a kv subject', () => {
     });
 
     it('refuses a pattern the grammar cannot enumerate from a key', async () => {
-        for (const subject of [
-            `kv:${OWN_APP}:ca*rt`,
-            `kv:${OWN_APP}:cart?`,
-        ])
+        for (const subject of [`kv:${OWN_APP}:ca*rt`, `kv:${OWN_APP}:cart?`])
             await expect(subscribeKv(subject)).rejects.toSatisfy(
                 (err: unknown) =>
                     isHttpError(err) && err.legacyCode === 'invalid_kv_pattern',
@@ -2329,8 +2388,7 @@ describe('delivering a kv change', () => {
 
     const eventFor = (subId: string) =>
         sent.find((one) => one.envelope.subId === subId)?.envelope.event as
-            | Record<string, unknown>
-            | undefined;
+            Record<string, unknown> | undefined;
 
     it('hands the value to the row that asked for it and to no other', async () => {
         vi.useFakeTimers();
@@ -2445,7 +2503,9 @@ describe('delivering a kv change', () => {
         await dispatchKv(['cart'], { values: ['kept'] });
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
-        const deliveries = sent.filter((one) => one.envelope.event.op === 'set');
+        const deliveries = sent.filter(
+            (one) => one.envelope.event.op === 'set',
+        );
         const gaps = sent.filter((one) => one.envelope.event.op === 'gap');
         expect(deliveries).toHaveLength(128);
         expect(gaps).toHaveLength(1);
@@ -2492,12 +2552,12 @@ describe('delivering a kv change', () => {
         await dispatchKv(['cart']);
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
-        expect(sent.filter((one) => one.envelope.event.op === 'set')).toHaveLength(
-            EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.limit,
-        );
-        expect(sent.filter((one) => one.envelope.event.op === 'gap')).toHaveLength(
-            1,
-        );
+        expect(
+            sent.filter((one) => one.envelope.event.op === 'set'),
+        ).toHaveLength(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.limit);
+        expect(
+            sent.filter((one) => one.envelope.event.op === 'gap'),
+        ).toHaveLength(1);
     });
 
     it('uses the free KV limit when owner policy lookup fails', async () => {
@@ -2508,7 +2568,9 @@ describe('delivering a kv change', () => {
         await dispatchKv(['cart']);
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
-        expect(sent.filter((one) => one.envelope.event.op === 'set')).toHaveLength(
+        expect(
+            sent.filter((one) => one.envelope.event.op === 'set'),
+        ).toHaveLength(
             EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.bySubscription[
                 DEFAULT_FREE_SUBSCRIPTION
             ],
@@ -2521,7 +2583,9 @@ describe('delivering a kv change', () => {
         await seedKvSubscriptions(
             EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT.bySubscription[
                 DEFAULT_FREE_SUBSCRIPTION
-            ] - 128 + 1,
+            ] -
+                128 +
+                1,
             { includeValue: true, match: 'elsewhere', tag: 'filtered-out' },
         );
 
@@ -2543,7 +2607,9 @@ describe('delivering a kv change', () => {
         });
         await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
 
-        const deliveries = sent.filter((one) => one.envelope.event.op === 'set');
+        const deliveries = sent.filter(
+            (one) => one.envelope.event.op === 'set',
+        );
         expect(deliveries).toHaveLength(128);
         expect(
             deliveries.every(
@@ -2769,8 +2835,7 @@ describe('what a kv dispatch costs', () => {
 });
 
 describe('the cross-app kv gate', () => {
-    const grantRead = (app: string) =>
-        grants.add(`app-data:${app}:kv:read`);
+    const grantRead = (app: string) => grants.add(`app-data:${app}:kv:read`);
 
     const crossAppService = () =>
         buildService({
@@ -2834,7 +2899,8 @@ describe('the cross-app kv gate', () => {
         ({ service } = crossAppService());
 
         await expect(subscribeKv(`kv:${OTHER_APP}:cart`)).rejects.toSatisfy(
-            (err: unknown) => isHttpError(err) && err.legacyCode === 'forbidden',
+            (err: unknown) =>
+                isHttpError(err) && err.legacyCode === 'forbidden',
         );
     });
 
@@ -2845,8 +2911,7 @@ describe('the cross-app kv gate', () => {
 
         await expect(subscribeKv('kv:app-gone:cart')).rejects.toSatisfy(
             (err: unknown) =>
-                isHttpError(err) &&
-                err.legacyCode === 'subject_does_not_exist',
+                isHttpError(err) && err.legacyCode === 'subject_does_not_exist',
         );
     });
 
@@ -2860,7 +2925,8 @@ describe('the cross-app kv gate', () => {
         });
 
         await expect(subscribeKv(`kv:${OTHER_APP}:cart`)).rejects.toSatisfy(
-            (err: unknown) => isHttpError(err) && err.legacyCode === 'forbidden',
+            (err: unknown) =>
+                isHttpError(err) && err.legacyCode === 'forbidden',
         );
     });
 
@@ -3135,9 +3201,7 @@ describe('cross-user kv handles', () => {
         expect(off.kvHandlesEnabled).toBe(false);
         expect(service.kvHandlesEnabled).toBe(true);
 
-        await expect(
-            subscribeAsGuest(`kv:${handle}:*`, off),
-        ).rejects.toSatisfy(
+        await expect(subscribeAsGuest(`kv:${handle}:*`, off)).rejects.toSatisfy(
             (err: unknown) =>
                 isHttpError(err) &&
                 err.legacyCode === 'events_kv_handles_disabled',
@@ -3345,10 +3409,13 @@ describe('cross-user kv handles', () => {
     it.each([
         ['an unknown handle', () => undefined],
         ['a revoked handle', () => mintHandle({ revokedAt: 1 })],
-        ['a handle whose grant the caller does not hold', () => {
-            mintHandle();
-            grants.delete(permission);
-        }],
+        [
+            'a handle whose grant the caller does not hold',
+            () => {
+                mintHandle();
+                grants.delete(permission);
+            },
+        ],
     ])('answers %s as absent', async (_case, setUp) => {
         setUp();
         await expect(subscribeAsGuest(`kv:${handle}:*`)).rejects.toSatisfy(

src/backend/services/events/appDeletion.integration.test.ts

@@ -19,19 +19,29 @@
 
 /**
  * What deleting an app leaves behind on the events side: its durable rows,
- * their backlog and its published handlers can never deliver again — the
- * grant identity they are re-checked under no longer resolves — so
- * `settleDeletedApp` tears them down on the `app.changed` bus key rather than
- * leaving them to cost their holder an anchor slot forever. Also the hourly
- * sweep that catches whatever neither this nor `workers.destroy` reached.
+ * their backlog and its published handlers can never deliver again — the grant
+ * identity they are re-checked under no longer resolves — so `settleDeletedApp`
+ * tears them down on the `app.changed` bus key rather than leaving them to cost
+ * their holder an anchor slot forever. Also the hourly sweep that catches
+ * whatever neither this nor `workers.destroy` reached.
  */
 
-import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import {
+    afterAll,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
 import { v4 as uuidv4 } from 'uuid';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import { EVENTS_WORKER_SESSION_NAME } from './workerRuntime.js';
+import { appHandlerScope } from '../../stores/events/EventHandlerStore.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 
@@ -125,10 +135,13 @@ const emitDeleted = async (appUid: string): Promise<void> => {
 const publish = (
     token: string,
     body: { appUid: string; name: string; source: string },
-): Promise<ApiResponse> => call('POST', '/events/handlers/publish', token, body);
+): Promise<ApiResponse> =>
+    call('POST', '/events/handlers/publish', token, body);
 
-/** A durable `single` subscription bound to a published handler, with the
- * grants a background delivery needs. Returns the subscribed subId. */
+/**
+ * A durable `single` subscription bound to a published handler, with the grants
+ * a background delivery needs. Returns the subscribed subId.
+ */
 const subscribeBackgroundHandler = async (
     app: { uid: string; token: string },
     handlerName: string,
@@ -162,6 +175,87 @@ const subscribeBackgroundHandler = async (
     return subscribed.body.subId as string;
 };
 
+describe('user scopes of a deleted app or account', () => {
+    const publishUserHandler = (
+        appUid: string,
+        holder: number,
+        name = 'mine',
+    ) =>
+        env.server.stores.eventHandler.publish({
+            scope: { appUid, holder },
+            name,
+            source: SOURCE,
+            broadcast: false,
+        });
+    const userHandlers = (appUid: string, holder: number) =>
+        env.server.stores.eventHandler.listForApp({ appUid, holder });
+
+    let destroyed: Array<{ appUid: string; scope: string; userId: number }>;
+    beforeAll(() => {
+        env.server.clients.event.on('events.worker.destroy', (_key, event) => {
+            destroyed?.push({
+                appUid: event.appUid,
+                scope: event.scope,
+                userId: event.actor.user.id,
+            });
+        });
+    });
+    beforeEach(async () => {
+        destroyed = [];
+        await env.server.clients.db.write(
+            'DELETE FROM `event_user_handlers`',
+            [],
+        );
+    });
+
+    it('removes every holder`s scope with the app, announcing each as the holder', async () => {
+        const app = await makeApp(userId);
+        await publishUserHandler(app.uid, userId);
+        await publishUserHandler(app.uid, otherUserId);
+        await publishUserHandler(app.uid, otherUserId, 'second');
+        const untouched = await makeApp(userId);
+        await publishUserHandler(untouched.uid, userId);
+
+        await emitDeleted(app.uid);
+
+        expect(await userHandlers(app.uid, userId)).toEqual([]);
+        expect(await userHandlers(app.uid, otherUserId)).toEqual([]);
+        expect(
+            destroyed
+                .filter((event) => event.scope === 'user')
+                .map((event) => [event.appUid, event.userId])
+                .sort(),
+        ).toEqual(
+            [
+                [app.uid, userId],
+                [app.uid, otherUserId],
+            ].sort(),
+        );
+        // Another app's scope is not touched.
+        expect(await userHandlers(untouched.uid, userId)).toHaveLength(1);
+    });
+
+    it('removes a deleted account`s scopes in every app, and no one else`s', async () => {
+        const first = await makeApp(userId);
+        const second = await makeApp(userId);
+        for (const app of [first, second]) {
+            await publishUserHandler(app.uid, otherUserId);
+            await publishUserHandler(app.uid, userId);
+        }
+
+        await env.server.clients.event.emitAndWait(
+            'user.delete',
+            { user_id: otherUserId, appUids: [] },
+            {},
+        );
+
+        for (const app of [first, second]) {
+            expect(await userHandlers(app.uid, otherUserId)).toEqual([]);
+            expect(await userHandlers(app.uid, userId)).toHaveLength(1);
+        }
+    });
+});
+
 describe('an app deleted while it has events state', () => {
     it('clears the rows, the backlog and the handlers when the app is deleted', async () => {
         const app = await makeApp(userId);
@@ -186,15 +280,17 @@ describe('an app deleted while it has events state', () => {
 
         await emitDeleted(app.uid);
 
-        const listed = await env.server.stores.durableSubscription.listForHolder(
-            userId,
-            { appUid: app.uid },
-        );
+        const listed =
+            await env.server.stores.durableSubscription.listForHolder(userId, {
+                appUid: app.uid,
+            });
         expect(listed.items).toEqual([]);
         expect(await env.server.stores.pendingDelivery.depth(subId)).toBe(0);
-        expect(await env.server.stores.eventHandler.listForApp(app.uid)).toEqual(
-            [],
-        );
+        expect(
+            await env.server.stores.eventHandler.listForApp(
+                appHandlerScope(app.uid),
+            ),
+        ).toEqual([]);
     });
 
     it("clears the rows and handlers of a deleted account's apps, and the account's subscription keys", async () => {
@@ -217,14 +313,16 @@ describe('an app deleted while it has events state', () => {
             {},
         );
 
-        const listed = await env.server.stores.durableSubscription.listForHolder(
-            userId,
-            { appUid: app.uid },
-        );
+        const listed =
+            await env.server.stores.durableSubscription.listForHolder(userId, {
+                appUid: app.uid,
+            });
         expect(listed.items).toEqual([]);
-        expect(await env.server.stores.eventHandler.listForApp(app.uid)).toEqual(
-            [],
-        );
+        expect(
+            await env.server.stores.eventHandler.listForApp(
+                appHandlerScope(app.uid),
+            ),
+        ).toEqual([]);
         expect(
             await env.server.stores.eventSubscription.isRegionWarm(goneUserId),
         ).toBe(false);
@@ -259,12 +357,10 @@ describe('an app deleted while it has events state', () => {
             expiresAt: null,
         });
 
-        const { actor: ownerActor } = await env.server.services.auth.authenticate(
-            env.users.user.token,
-        );
-        const { actor: otherActor } = await env.server.services.auth.authenticate(
-            env.users.other.token,
-        );
+        const { actor: ownerActor } =
+            await env.server.services.auth.authenticate(env.users.user.token);
+        const { actor: otherActor } =
+            await env.server.services.auth.authenticate(env.users.other.token);
         const ownerToken = await env.server.services.auth.createWorkerAppToken(
             ownerActor!,
             app.uid,
@@ -327,7 +423,9 @@ describe('an app deleted while it has events state', () => {
         await env.server.stores.app.delete((row as { id: number }).id);
 
         const revoked =
-            await env.server.services.events.sweepStrandedWorkerSessions();
+            await env.server.services.events.sweepStrandedWorkerSessions(
+                testJobContext(),
+            );
         expect(revoked).toBeGreaterThanOrEqual(1);
 
         // `authenticate()` can no longer resolve the token at all — the app
@@ -366,9 +464,60 @@ describe('an app deleted while it has events state', () => {
             EVENTS_WORKER_SESSION_NAME,
         );
 
-        await env.server.services.events.sweepStrandedWorkerSessions();
+        await env.server.services.events.sweepStrandedWorkerSessions(
+            testJobContext(),
+        );
 
         const reauth = await env.server.services.auth.authenticate(token);
         expect(reauth).not.toHaveProperty('reauth');
     });
+
+    it('reaches a stranded session past one run`s two pages on the next run, then wraps', async () => {
+        const db = env.server.clients.db;
+        const now = Math.floor(Date.now() / 1000);
+        const meta = JSON.stringify({
+            worker_name: EVENTS_WORKER_SESSION_NAME,
+        });
+        const workerSession = (appUid: string) => ({
+            statement:
+                'INSERT INTO `sessions` (`uuid`, `user_id`, `meta`, `last_activity`, `created_at`, `kind`, `app_uid`) ' +
+                "VALUES (?, ?, ?, ?, ?, 'worker', ?)",
+            values: [uuidv4(), userId, meta, now, now, appUid],
+        });
+        // A run reads two pages of 500; these fill them, all still standing.
+        const live = Array.from(
+            { length: 1000 },
+            (_, i) => `app-live-${uuidv4().slice(0, 8)}-${i}`,
+        );
+        await db.batchWrite(
+            live.map((uid) => ({
+                statement:
+                    'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
+                values: [uid, uid, uid, `https://${uid}.example/`, userId],
+            })),
+        );
+        await db.batchWrite(live.map(workerSession));
+        const stranded = workerSession(`app-gone-${uuidv4()}`);
+        await db.batchWrite([stranded]);
+        const strandedRevoked = async () => {
+            const [row] = await db.pread(
+                'SELECT `revoked_at` FROM `sessions` WHERE `uuid` = ?',
+                [stranded.values[0]],
+            );
+            return row.revoked_at !== null;
+        };
+
+        const first = testJobContext();
+        await env.server.services.events.sweepStrandedWorkerSessions(first);
+        expect(await strandedRevoked()).toBe(false);
+        expect(first.saved.at(-1)).toMatchObject({
+            afterId: expect.any(Number),
+        });
+
+        const second = testJobContext(first.saved.at(-1));
+        await env.server.services.events.sweepStrandedWorkerSessions(second);
+        expect(await strandedRevoked()).toBe(true);
+        // The end of the table wraps the cursor back to the start.
+        expect(second.saved.at(-1)).toBeNull();
+    });
 });

src/backend/services/events/durable.integration.test.ts

@@ -872,6 +872,71 @@ describe('what a write costs once a durable row exists', () => {
 
         expect(deliveryOf(path)).toBeUndefined();
     });
+
+    it('publishes the expiry reap`s bump, so a region still caching the row drops it', async () => {
+        await clearRows();
+        const created = await subscribe(env.users.user.token, {
+            expiresAt: Math.floor(Date.now() / 1000) + 3600,
+        });
+        const subId = created.body.subId as string;
+        const stale =
+            await env.server.stores.durableSubscription.getBySubId(subId);
+        await env.server.clients.db.write(
+            `UPDATE \`${TABLE}\` SET \`expires_at\` = ? WHERE \`sub_id\` = ?`,
+            [Math.floor(Date.now() / 1000) - 1, subId],
+        );
+
+        const heard: Array<Record<string, unknown>> = [];
+        const listener = (_key: string, data: unknown) => {
+            heard.push(data as Record<string, unknown>);
+        };
+        const announced = vi.spyOn(
+            env.server.services.eventForward,
+            'announceGeneration',
+        );
+        env.server.clients.event.on(
+            'outer.pubsub.events.generationBumped',
+            listener as never,
+        );
+        try {
+            await expect(events().sweepExpired()).resolves.toBe(1);
+            // Over the addressed peer channel as well as the bus.
+            expect(announced).toHaveBeenCalledWith(
+                expect.objectContaining({ userId }),
+                true,
+                'subscription',
+            );
+        } finally {
+            env.server.clients.event.off(
+                'outer.pubsub.events.generationBumped',
+                listener as never,
+            );
+            announced.mockRestore();
+        }
+        const published = heard.find(
+            (bump) => bump.userId === userId && bump.durable === true,
+        );
+        expect(published).toBeDefined();
+
+        // Another region's Redis still holds the row; this one stands in for it.
+        await env.server.stores.eventSubscription.cacheDurable([stale!]);
+        delivered.length = 0;
+        const before = `${anchor}/stale-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: before });
+        await settle(before);
+        expect(deliveryOf(before)?.subId).toBe(subId);
+
+        await env.server.clients.event.emitAndWait(
+            'outer.pubsub.events.generationBumped',
+            published!,
+            { from_outside: true },
+        );
+        delivered.length = 0;
+        const after = `${anchor}/peer-swept-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: after });
+        await quiet();
+        expect(deliveryOf(after)).toBeUndefined();
+    });
 });
 
 describe('where a durable delivery is addressed', () => {

src/backend/services/events/fetch.integration.test.ts

@@ -616,6 +616,210 @@ describe('reading again from a saved cursor', () => {
     });
 });
 
+describe('starting from the end', () => {
+    const SUBJECT = 'notif:app-user';
+
+    const message = async (title: string, inApp = appUid): Promise<string> =>
+        (
+            await env.server.stores.notification.create({
+                userId,
+                value: { type: 'app.message', title },
+                type: 'app.message',
+                audience: 'app-user',
+                appUid: inApp,
+                group: `group-${uuidv4()}`,
+            })
+        ).uid as string;
+
+    const fromEnd = async (token = appToken): Promise<string> => {
+        const end = await fetchPage(token, { subject: SUBJECT, from: 'end' });
+        expect(end.status).toBe(200);
+        expect(end.body.items).toEqual([]);
+        expect(end.body.cursor).toBeTruthy();
+        return end.body.cursor!;
+    };
+
+    it('answers a cursor on an empty slice that returns what comes after', async () => {
+        await clearMailboxes();
+        const cursor = await fromEnd();
+
+        const later = await message('later');
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual([later]);
+    });
+
+    it('skips what is there and returns only newer rows', async () => {
+        await clearMailboxes();
+        for (let i = 0; i < 3; i++) await message(`before ${i}`);
+        const cursor = await fromEnd();
+
+        const quiet = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(quiet.body.items).toEqual([]);
+        expect(quiet.body.cursor).toBeUndefined();
+
+        const later = [await message('after 0'), await message('after 1')];
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual(later);
+    });
+
+    it('returns a row replaced after the cursor was taken', async () => {
+        await clearMailboxes();
+        const old = await message('old');
+        await message('older still unchanged');
+        const cursor = await fromEnd();
+
+        await env.server.services.notification.notifyUpdate(
+            old,
+            userId,
+            { title: 'replaced' },
+            { type: 'app.message', replace: true },
+        );
+
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual([old]);
+        expect(next.body.items![0]).toMatchObject({
+            replaced: true,
+            notification: { title: 'replaced' },
+        });
+    });
+
+    it('returns a replacement stamped by a clock behind the server`s', async () => {
+        await clearMailboxes();
+        const behind = Date.now() - 60_000;
+        const [early, late] = [await message('early'), await message('late')];
+        const replace = (uid: string, title: string, replacedAt: number) =>
+            env.server.stores.notification.updateValue(
+                uid,
+                userId,
+                { type: 'app.message', title },
+                { onlyUnseen: true, replacedAt },
+            );
+        await replace(early, 'replaced before', behind - 1_000);
+        const cursor = await fromEnd();
+
+        await replace(late, 'replaced after', behind);
+
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual([late]);
+        expect(next.body.items![0]).toMatchObject({
+            notification: { title: 'replaced after' },
+        });
+    });
+
+    it('keeps to the slice the subject names', async () => {
+        await clearMailboxes();
+        await message('own before');
+        const cursor = await fromEnd();
+
+        await message('other app', otherAppUid);
+        await seed(userId, { title: 'account row' });
+        const own = await message('own after');
+
+        const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
+        expect(uidsOf(next)).toEqual([own]);
+    });
+
+    it('answers no cursor for a slice the caller cannot see', async () => {
+        await clearMailboxes();
+        await message('other app', otherAppUid);
+
+        for (const [token, subject] of [
+            [appToken, `notif:${otherAppUid}:app-user`],
+            [appToken, `notif:${userUuid}:account`],
+            [appReadUrlToken, SUBJECT],
+        ] as const) {
+            const page = await fetchPage(token, { subject, from: 'end' });
+            expect(page.status).toBe(200);
+            expect(page.body.items).toEqual([]);
+            expect(page.body.cursor).toBeUndefined();
+        }
+    });
+
+    it('reads from the cursor when one is given', async () => {
+        await clearMailboxes();
+        const uids = [await message('m0'), await message('m1')];
+        const first = await fetchPage(appToken, { subject: SUBJECT, limit: 1 });
+
+        const page = await fetchPage(appToken, {
+            subject: SUBJECT,
+            cursor: first.body.cursor,
+            from: 'end',
+        });
+        expect(uidsOf(page)).toEqual([uids[1]]);
+    });
+
+    it('rejects a `from` that is neither start nor end', async () => {
+        const page = await fetchPage(appToken, {
+            subject: SUBJECT,
+            from: 'middle',
+        });
+        expect(page.status).toBe(400);
+
+        const start = await fetchPage(appToken, {
+            subject: SUBJECT,
+            from: 'start',
+        });
+        expect(start.status).toBe(200);
+    });
+
+    it('starts the SDK feed without catch-up in one read', async () => {
+        await clearMailboxes();
+        for (let i = 0; i < 3; i++) await message(`before ${i}`);
+
+        const appActor = (await env.server.services.auth.authenticate(appToken))
+            .actor!;
+        const reads: Array<Record<string, unknown>> = [];
+        let receive: (delivery: { event: unknown }) => void = () => {};
+        const titles: string[] = [];
+        const feed = new NotificationFeed(
+            {
+                authToken: appToken,
+                events: {
+                    onLocal: async (
+                        _subject: string,
+                        handler: typeof receive,
+                    ) => {
+                        receive = handler;
+                        return { off: async () => {} };
+                    },
+                    fetch: async (options: Record<string, unknown>) => {
+                        reads.push(options);
+                        return env.server.services.events.fetchMissed(
+                            appActor,
+                            options,
+                        );
+                    },
+                },
+            } as never,
+            (notification: { title: string | null }) => {
+                titles.push(String(notification.title));
+            },
+            { catchUp: false },
+        );
+
+        try {
+            await feed.start();
+            expect(reads).toEqual([
+                { subject: SUBJECT, limit: 200, from: 'end' },
+            ]);
+
+            await message('after start');
+            receive({ event: { op: 'gap', reason: 'reconnect' } });
+            await vi.waitFor(() => expect(reads).toHaveLength(2));
+            await new Promise((resolve) => setTimeout(resolve, 0));
+        } finally {
+            await feed.off();
+        }
+
+        expect(reads[1]).toEqual({
+            subject: SUBJECT,
+            limit: 200,
+            cursor: expect.any(String),
+        });
+        expect(titles).toEqual(['after start']);
+    });
+});
+
 describe('cross-user isolation', () => {
     it('returns nothing of another account, whatever the subject names', async () => {
         await clearMailboxes();

src/backend/services/events/forwardQueue.test.ts

@@ -217,6 +217,97 @@ describe('the addressed peer queue', () => {
         ).toEqual(['b']);
     });
 
+    it.each([false, true])(
+        'waits for an active send and ships its tail after failure %s',
+        async (failFirst) => {
+            let release!: () => void;
+            const held = new Promise<void>((resolve) => {
+                release = resolve;
+            });
+            let releaseTail!: () => void;
+            const tailHeld = new Promise<void>((resolve) => {
+                releaseTail = resolve;
+            });
+            const sent: ForwardItem[][] = [];
+            const { queue } = makeQueue({
+                maxItems: 1,
+                flushMs: 60_000,
+                send: async (_peerId, items) => {
+                    sent.push(items);
+                    if (sent.length === 1) {
+                        await held;
+                        if (failFirst) throw new Error('peer unavailable');
+                    } else {
+                        await tailHeld;
+                    }
+                },
+            });
+            queue.push('east', delivery('first'));
+            await vi.waitFor(() => expect(sent).toHaveLength(1));
+            queue.push('east', delivery('tail'));
+            let finished = false;
+            const draining = queue.flushAll().then(() => {
+                finished = true;
+            });
+            try {
+                await new Promise((resolve) => setTimeout(resolve, 20));
+                expect(finished).toBe(false);
+                release();
+                await vi.waitFor(() => expect(sent).toHaveLength(2));
+                expect(finished).toBe(false);
+            } finally {
+                release();
+                releaseTail();
+                await draining;
+                queue.stop();
+            }
+            expect(sent).toEqual([[delivery('first')], [delivery('tail')]]);
+        },
+    );
+
+    it('releases the peer after a send throws synchronously', async () => {
+        const send = vi.fn(() => {
+            throw new Error('send failed');
+        });
+        const { queue } = makeQueue({ maxItems: 1, send });
+        queue.push('east', delivery('first'));
+        await queue.flushAll();
+        queue.push('east', delivery('second'));
+        await queue.flushAll();
+        expect(send).toHaveBeenCalledTimes(2);
+    });
+
+    it('bounds a final drain while a send is hung', async () => {
+        vi.useFakeTimers();
+        let release!: () => void;
+        const held = new Promise<void>((resolve) => {
+            release = resolve;
+        });
+        const send = vi.fn(async () => {
+            await held;
+        });
+        const { queue } = makeQueue({ maxItems: 1, send });
+        try {
+            queue.push('east', delivery('first'));
+            queue.push('east', delivery('tail'));
+            let finished = false;
+            const draining = queue.flushAll().then(() => {
+                finished = true;
+            });
+            await vi.advanceTimersByTimeAsync(4_999);
+            expect(finished).toBe(false);
+            await vi.advanceTimersByTimeAsync(1);
+            await draining;
+            queue.stop();
+            release();
+            await vi.advanceTimersByTimeAsync(100);
+            expect(send).toHaveBeenCalledTimes(1);
+            expect(vi.getTimerCount()).toBe(0);
+        } finally {
+            release();
+            vi.useRealTimers();
+        }
+    });
     it('sends nothing more once it is stopped', async () => {
         const { queue, sent } = makeQueue();
         queue.stop();

src/backend/services/events/forwardQueue.ts

@@ -18,6 +18,8 @@
  */
 
 import type { EventKey, KvOp } from '../../clients/event/types.js';
+import { settleWithin } from '../../util/concurrency.js';
+import { FINAL_FLUSH_TIMEOUT_MS } from '../../util/gracefulShutdown.js';
 import type { DeliverableEvent } from './registry.js';
 
 /**
@@ -86,7 +88,7 @@ export interface ForwardWatch {
 /** One committed change, replayed against another region's session rows. */
 export interface ForwardEvent {
     kind: 'event';
-    family: 'fs' | 'kv';
+    family: 'fs' | 'kv' | 'notif';
     ownerUserId: number;
     /** Who caused it, so `self` is right on the far side. */
     actingUserId?: number;
@@ -108,6 +110,17 @@ export interface ForwardEvent {
             ancestors: Array<{ uid: string; path: string }>;
         };
     };
+    notif?: {
+        userUuid: string;
+        uid: string;
+        type: string;
+        audience: string;
+        appUid: string | null;
+        value: Record<string, unknown>;
+        dismissed?: true;
+        replaced?: true;
+        handlerDepth?: number;
+    };
     kv?: {
         userUuid: string;
         appUid: string;
@@ -204,7 +217,7 @@ interface PeerQueue {
     items: ForwardItem[];
     bytes: number;
     timer: ReturnType<typeof setTimeout> | null;
-    flushing: boolean;
+    flushing: Promise<void> | null;
 }
 
 export class PeerForwardQueue {
@@ -272,19 +285,27 @@ export class PeerForwardQueue {
     }
 
     /** Ship whatever is held for one peer. */
-    async flush(peerId: string): Promise<void> {
+    flush(peerId: string): Promise<void> {
         const queue = this.#peers.get(peerId);
-        if (!queue || queue.flushing || queue.items.length === 0) return;
+        if (!queue) return Promise.resolve();
+        if (queue.flushing) return queue.flushing;
+        if (queue.items.length === 0) return Promise.resolve();
         if (queue.timer) {
             clearTimeout(queue.timer);
             queue.timer = null;
         }
 
-        queue.flushing = true;
+        queue.flushing = this.#send(peerId, queue);
+        return queue.flushing;
+    }
+
+    async #send(peerId: string, queue: PeerQueue): Promise<void> {
         const batch = queue.items;
         queue.items = [];
         queue.bytes = 0;
         try {
+            // Register the in-flight promise before a send can throw synchronously.
+            await Promise.resolve();
             await this.#options.send(peerId, batch);
         } catch (err) {
             // A lost batch is a lost at-most-once delivery, and a `single`
@@ -292,7 +313,7 @@ export class PeerForwardQueue {
             // hand the client an event whose moment has passed.
             console.warn(`[events] forward to ${peerId} failed`, err);
         } finally {
-            queue.flushing = false;
+            queue.flushing = null;
             // Anything that arrived mid-flush gets the next window.
             if (queue.items.length > 0 && !queue.timer && !this.#stopped) {
                 queue.timer = setTimeout(() => {
@@ -305,7 +326,16 @@ export class PeerForwardQueue {
     }
 
     async flushAll(): Promise<void> {
-        await Promise.all([...this.#peers.keys()].map((id) => this.flush(id)));
+        const deadline = Date.now() + FINAL_FLUSH_TIMEOUT_MS;
+        const work = Promise.allSettled(
+            [...this.#peers.entries()].map(async ([id, queue]) => {
+                await queue.flushing;
+                if (Date.now() < deadline) await this.flush(id);
+            }),
+        );
+        if (!(await settleWithin(work, FINAL_FLUSH_TIMEOUT_MS))) {
+            console.warn('[events] final flush timed out');
+        }
     }
 
     stop(): void {
@@ -323,7 +353,7 @@ export class PeerForwardQueue {
             items: [],
             bytes: 0,
             timer: null,
-            flushing: false,
+            flushing: null,
         };
         this.#peers.set(peerId, queue);
         return queue;
@@ -349,7 +379,7 @@ const isGapMarker = (item: ForwardItem): boolean =>
  */
 const shed = (queue: PeerQueue, count: number): ForwardItem[] => {
     const dropped: ForwardItem[] = [];
-    for (let i = 0; i < queue.items.length && dropped.length < count;) {
+    for (let i = 0; i < queue.items.length && dropped.length < count; ) {
         if (isGapMarker(queue.items[i])) {
             i++;
             continue;
@@ -372,7 +402,7 @@ const shed = (queue: PeerQueue, count: number): ForwardItem[] => {
 const shedBytes = (queue: PeerQueue, maxBytesHeld: number): ForwardItem[] => {
     const dropped: ForwardItem[] = [];
     let remaining = queue.bytes;
-    for (let i = 0; i < queue.items.length && remaining > maxBytesHeld;) {
+    for (let i = 0; i < queue.items.length && remaining > maxBytesHeld; ) {
         if (isGapMarker(queue.items[i])) {
             i++;
             continue;

src/backend/services/events/forwarding.test.ts

@@ -79,6 +79,14 @@ vi.mock(import('@opentelemetry/api'), async (importOriginal) => ({
                     metricCalls.push({ name, value, attributes });
                 },
             }),
+            createHistogram: (name: string) => ({
+                record: (
+                    value: number,
+                    attributes: Record<string, unknown> = {},
+                ) => {
+                    metricCalls.push({ name, value, attributes });
+                },
+            }),
         }),
     },
 }));
@@ -394,7 +402,7 @@ const makeRegion = (
                 fields: { code: 'subject_does_not_exist' },
             }),
         },
-        notification: { notify: vi.fn() },
+        notification: { notify: vi.fn(), deliverOverSocket: vi.fn() },
         metering: {
             bufferIncrementUsages: (
                 actor: Actor,
@@ -441,6 +449,7 @@ const makeRegion = (
             ): Promise<{
                 row: DurableSubscription;
                 bump: { userId: number; generation: number };
+                reaped: [];
             }> => {
                 const row = durableRow({
                     holderUserId: input.holderUserId as number,
@@ -469,6 +478,7 @@ const makeRegion = (
                             row.ownerUserId,
                         ),
                     },
+                    reaped: [],
                 };
             },
         },
@@ -2391,3 +2401,68 @@ describe('a durable row created in another region', () => {
         );
     });
 });
+
+
+describe('notification dismissals across connected pages', () => {
+    const config = { events: { enabled: true, notificationsFoldIn: true } } as Partial<IConfig>;
+    const appUid = 'notification-app';
+    const input = (uid = 'one') => ({
+        userId, userUuid: `user-${userId}`, uid,
+        type: 'app.message', audience: 'app-user', appUid,
+        value: {}, createdAt: Date.now(), dismissed: true as const,
+    });
+
+    it('reaches local and remote session pages without content or a forwarding loop', async () => {
+        const west = makeRegion('west', ['east'], config);
+        const east = makeRegion('east', ['west'], config);
+        for (const [region, socket] of [[west, 'local'], [east, 'remote']] as const) {
+            await region.events.subscribe(actorFor(appUid), socket, { subject: 'notif:app-user' });
+        }
+        await posted(east);
+        await quiet(100);
+        await west.events.dispatchNotif({ ...input(), value: { title: 'private' } });
+        await arrived(west);
+        await arrived(east);
+        for (const region of [west, east]) {
+            expect(region.sent).toHaveLength(1);
+            expect(region.sent[0].event).toMatchObject({ op: 'dismiss', uid: 'one', appUid });
+            expect(region.sent[0].event).not.toHaveProperty('notification');
+        }
+        expect(east.sent[0].event.id).toBe(west.sent[0].event.id);
+        const forwarded = west.posts.flatMap((p) => p.batch.items)
+            .find((item): item is ForwardEvent => item.kind === 'event');
+        expect(forwarded?.notif?.value).toEqual({});
+        expect(east.posts.flatMap((p) => p.batch.items).filter((i) => i.kind === 'event')).toEqual([]);
+    });
+
+    it('keeps separate dismissed ids in the same coalescing window', async () => {
+        const region = makeRegion('west', [], config);
+        await region.events.subscribe(actorFor(appUid), 'local', { subject: 'notif:app-user' });
+        await region.events.dispatchNotif(input('one'));
+        await region.events.dispatchNotif(input('two'));
+        await arrived(region, 2);
+        expect(region.sent.map((item) => 'uid' in item.event && item.event.uid)).toEqual(['one', 'two']);
+    });
+
+    it('does not deliver another app, account audience or another user to an app page', async () => {
+        const region = makeRegion('west', [], config);
+        await region.events.subscribe(actorFor(appUid), 'local', { subject: 'notif:app-user' });
+        await region.events.dispatchNotif({ ...input(), appUid: 'other-app' });
+        await region.events.dispatchNotif({ ...input(), audience: 'account' });
+        await region.events.dispatchNotif({ ...input(), userId: userId + 1, userUuid: 'other-user' });
+        await quiet(350);
+        expect(region.sent).toEqual([]);
+    });
+
+    it('also forwards postings through the same session path', async () => {
+        const west = makeRegion('west', ['east'], config);
+        const east = makeRegion('east', ['west'], config);
+        await east.events.subscribe(actorFor(appUid), 'remote', { subject: 'notif:app-user' });
+        await posted(east);
+        await quiet(100);
+        const { dismissed, ...postedInput } = input();
+        await west.events.dispatchNotif({ ...postedInput, value: { title: 'Hello' } });
+        await arrived(east);
+        expect(east.sent[0].event).toMatchObject({ op: 'post', uid: 'one', notification: { title: 'Hello' } });
+    });
+});

src/backend/services/events/handlers.integration.test.ts

@@ -45,7 +45,10 @@ import {
     EVENTS_WORKER_SOURCE_MAX_BYTES,
 } from '../../controllers/events/limits.js';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
-import { hashContent } from '../../stores/events/EventHandlerStore.js';
+import {
+    appHandlerScope,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import type { DurableSubscriptionView } from './EventsService.js';
@@ -220,7 +223,8 @@ afterAll(async () => {
 
 beforeEach(async () => {
     await env.server.clients.db.write('DELETE FROM `event_handlers`', []);
-    for (const row of await durable().listActiveForHolder(userId, null))
+    for (const row of (await durable().listActiveForHolderPage(userId, null, 0))
+        .rows)
         await durable().remove(row);
     await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []);
     await env.server.stores.eventSubscription.rebuildDurable(userId, []);
@@ -531,7 +535,7 @@ describe('what a publish reads', () => {
                 if (sql.includes(BY_NAME_SQL) && !landed) {
                     landed = true;
                     await env.server.stores.eventHandler.publish({
-                        appUid,
+                        scope: appHandlerScope(appUid),
                         name: 'ingestUpload',
                         source: NEXT_SOURCE,
                         replace: true,
@@ -589,7 +593,7 @@ describe('what a publish reads', () => {
             pread.mockRestore();
         }
         const standing = await env.server.stores.eventHandler.getByName(
-            appUid,
+            appHandlerScope(appUid),
             'ingestUpload',
         );
         expect(standing?.sourceHash).toBe(hashContent(THIRD_SOURCE));
@@ -602,7 +606,7 @@ describe('what a publish reads', () => {
             if (sql.includes(BY_NAME_SQL) && !beaten) {
                 beaten = true;
                 await env.server.stores.eventHandler.publish({
-                    appUid,
+                    scope: appHandlerScope(appUid),
                     name: 'ingestUpload',
                     source: THIRD_SOURCE,
                 });
@@ -658,7 +662,7 @@ describe('what a publish reads', () => {
             pread.mockRestore();
         }
         await expect(
-            env.server.stores.eventHandler.countForApp(appUid),
+            env.server.stores.eventHandler.countForApp(appHandlerScope(appUid)),
         ).resolves.toBe(0);
     });
 
@@ -777,7 +781,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(404);
         expect(refused.body.code).toBe('events_handler_not_found');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('binds an inline body whose hash is what is published', async () => {
@@ -805,7 +811,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(409);
         expect(refused.body.code).toBe('events_handler_hash_mismatch');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('still needs a name for a subscription owed to one consumer', async () => {

src/backend/services/events/kvHandleRevoke.integration.test.ts

@@ -383,7 +383,7 @@ describe('revoking a handle', () => {
         const sub = await subscribeDurable(handle);
 
         const store = env.server.stores.durableSubscription;
-        const byHolder = vi.spyOn(store, 'listActiveForHolder');
+        const byHolder = vi.spyOn(store, 'listActiveForHolderPage');
         const settle = vi.spyOn(events(), 'settleRevokedGrant');
 
         try {
@@ -400,7 +400,9 @@ describe('revoking a handle', () => {
             for (const call of byHolder.mock.calls)
                 expect(call.slice(1)).toEqual([
                     null,
-                    { includeSuspended: true },
+                    0,
+                    undefined,
+                    { includeSuspended: true, excludeSchedules: true },
                 ]);
         } finally {
             byHolder.mockRestore();
@@ -442,7 +444,9 @@ describe('revoking a handle', () => {
         const page = await shares().listKvHandles(owner.actor, {
             limit: 100,
         });
-        const deeperRow = page.items.find((one) => one.handle === deeper.handle);
+        const deeperRow = page.items.find(
+            (one) => one.handle === deeper.handle,
+        );
         expect(deeperRow?.revokedAt).toBeTypeOf('number');
         expect((await countLive()).total).toBe(before.total);
     });

src/backend/services/events/metering.test.ts

@@ -36,13 +36,17 @@ import {
     DurableQuotaRaceLost,
     type DurableSubscriptionInput,
 } from '../../stores/events/DurableSubscriptionStore.js';
-import type { DurableSubscription } from '../../stores/events/types.js';
+import {
+    scheduleToken,
+    type DurableSubscription,
+} from '../../stores/events/types.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
 import {
     DEFAULT_FREE_SUBSCRIPTION,
     DEFAULT_TEMP_SUBSCRIPTION,
 } from '../metering/consts.js';
 import type { UsageInput } from '../metering/types.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_COSTS } from './costs.js';
 import { EventsService } from './EventsService.js';
@@ -62,6 +66,12 @@ let solvent: Set<number>;
 /** Set to have the next create lose the last slot after its insert. */
 let raceLost: boolean;
 let emitted: Array<{ key: string; data: unknown }>;
+/** Rows one credit-sweep page takes; null for the service's own batch. */
+let pageSize: number | null;
+/** Rows one holder page takes; null for all of them in one page. */
+let holderPageSize: number | null;
+/** What `services.schedules` hears about changed schedule rows. */
+let schedulesChanged: (rows: readonly DurableSubscription[]) => void;
 
 const anchorUid = (): string => `docs-${seq}`;
 const anchorPath = (): string => `/u${userId}/Documents`;
@@ -99,7 +109,10 @@ const actorFor = (over: Partial<Actor> = {}): Actor =>
     }) as unknown as Actor;
 
 const appActor = (appUid: string): Actor =>
-    actorFor({ app: { uid: appUid, id: 1 }, effectiveApp: { uid: appUid, id: 1 } });
+    actorFor({
+        app: { uid: appUid, id: 1 },
+        effectiveApp: { uid: appUid, id: 1 },
+    });
 
 const anchorEntry = (): FSEntry =>
     ({
@@ -124,6 +137,9 @@ beforeEach(() => {
     solvent = new Set([userId]);
     raceLost = false;
     emitted = [];
+    pageSize = null;
+    holderPageSize = null;
+    schedulesChanged = () => undefined;
 
     service = new EventsService(
         { events: { enabled: true } } as IConfig,
@@ -155,26 +171,49 @@ beforeEach(() => {
                     return {
                         row,
                         bump: { userId: input.ownerUserId, generation: 1 },
+                        reaped: [],
                     };
                 },
-                listSuspendedPage: async (reason: string, afterId: number) =>
-                    afterId === 0
-                        ? {
-                              rows: rows.filter(
-                                  (row) => row.suspendedReason === reason,
-                              ),
-                              nextId: null,
-                          }
-                        : { rows: [], nextId: null },
-                listSuspendedForHolder: async (
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedPage: async (
+                    reason: string,
+                    afterId: number,
+                    batchSize: number,
+                ) => {
+                    const size = pageSize ?? batchSize;
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId && row.suspendedReason === reason,
+                        )
+                        .slice(0, size);
+                    return {
+                        rows: page.map(({ row }) => row),
+                        nextId: page.length < size ? null : page.at(-1)!.id,
+                    };
+                },
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedForHolderPage: async (
                     holderUserId: number,
                     reason: string,
-                ) =>
-                    rows.filter(
-                        (row) =>
-                            row.holderUserId === holderUserId &&
-                            row.suspendedReason === reason,
-                    ),
+                    afterId: number,
+                ) => {
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId &&
+                                row.holderUserId === holderUserId &&
+                                row.suspendedReason === reason,
+                        );
+                    const size = holderPageSize ?? page.length;
+                    const taken = page.slice(0, size);
+                    return {
+                        rows: taken.map(({ row }) => row),
+                        nextId: page.length > size ? taken.at(-1)!.id : null,
+                    };
+                },
                 resume: async (resuming: readonly DurableSubscription[]) => {
                     for (const row of resuming) {
                         resumed.push(row.subId);
@@ -233,10 +272,7 @@ beforeEach(() => {
             // plan allows, not about who agreed to it.
             permission: { check: async () => true },
             metering: {
-                bufferIncrementUsages: (
-                    actor: Actor,
-                    usages: UsageInput[],
-                ) => {
+                bufferIncrementUsages: (actor: Actor, usages: UsageInput[]) => {
                     for (const usage of usages)
                         metered.push({
                             userUuid: actor.user?.uuid,
@@ -247,6 +283,11 @@ beforeEach(() => {
                     solvent.has(Number(actor.user?.id ?? -1)),
                 getActorSubscription: async () => ({ id: plan }),
             },
+            schedules: {
+                onSubscriptionsChanged: async (
+                    changed: readonly DurableSubscription[],
+                ) => schedulesChanged(changed),
+            },
         } as never,
     );
 });
@@ -258,23 +299,105 @@ describe('a suspension waiting on a balance', () => {
             suspendedReason: 'no_credit',
         });
 
-    it('comes back once the account has budget again', async () => {
+    it('comes back once the account has budget again, and every region hears it', async () => {
         const row = suspended();
         rows.push(row);
 
-        await expect(service.sweepNoCredit()).resolves.toBe(1);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(1);
         expect(resumed).toEqual([row.subId]);
         expect(row.suspendedReason).toBeNull();
+        expect(emitted).toContainEqual({
+            key: 'outer.pubsub.events.generationBumped',
+            data: { userId, generation: 1, durable: true },
+        });
     });
 
     it('stays out of service while the account still has none', async () => {
         rows.push(suspended());
         solvent.clear();
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 
+    it('reaches rows past one run`s page cap across runs, then wraps', async () => {
+        // One row per page, so a run's 200 pages stop at 200 rows.
+        pageSize = 1;
+        for (let i = 0; i < 450; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 10_000 + i;
+        // Only the last holder has a balance again.
+        solvent = new Set([10_000 + 449]);
+
+        const first = testJobContext();
+        await expect(service.sweepNoCredit(first)).resolves.toBe(0);
+        expect(first.saved.at(-1)).toEqual({ afterId: 200 });
+
+        const second = testJobContext(first.saved.at(-1));
+        await expect(service.sweepNoCredit(second)).resolves.toBe(0);
+        expect(second.saved.at(-1)).toEqual({ afterId: 400 });
+
+        const third = testJobContext(second.saved.at(-1));
+        await expect(service.sweepNoCredit(third)).resolves.toBe(1);
+        expect(resumed).toEqual([rows[449].subId]);
+        // The end of the table wraps the cursor back to the start.
+        expect(third.saved.at(-1)).toBeNull();
+    });
+
+    it('stops between pages once its signal aborts, keeping the last full page', async () => {
+        pageSize = 1;
+        for (let i = 0; i < 5; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 20_000 + i;
+        const ctx = testJobContext();
+        const save = ctx.saveState;
+        ctx.saveState = async (next) => {
+            await save(next);
+            if (ctx.saved.length === 2) ctx.abort();
+        };
+
+        await service.sweepNoCredit(ctx);
+        expect(ctx.saved).toEqual([{ afterId: 1 }, { afterId: 2 }]);
+    });
+
+    it('resumes no later holder page once its signal aborts inside a page`s hook, keeping the checkpoint', async () => {
+        // A full sweep page, so finishing it would move the checkpoint.
+        pageSize = 3;
+        holderPageSize = 1;
+        for (let i = 0; i < 3; i++)
+            rows.push(
+                durableRow({
+                    subId: `sched-${i}`,
+                    token: scheduleToken(`schedule-${seq}-${i}`),
+                    suspendedAt: Math.floor(Date.now() / 1000),
+                    suspendedReason: 'no_credit',
+                }),
+            );
+        const ctx = testJobContext();
+        const heard: string[][] = [];
+        schedulesChanged = (changed) => {
+            heard.push(changed.map((row) => row.subId));
+            ctx.abort();
+        };
+
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(1);
+        expect(resumed).toEqual(['sched-0']);
+        expect(heard).toEqual([['sched-0']]);
+        expect(rows[1].suspendedReason).toBe('no_credit');
+        expect(ctx.saved).toEqual([]);
+    });
+
+    it('wraps a saved cursor past the end of a shrunken table back to the start', async () => {
+        rows.push(suspended());
+        const ctx = testJobContext({ afterId: 999 });
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([null]);
+    });
+
+    it('writes no cursor while there is nothing to page through', async () => {
+        const ctx = testJobContext();
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([]);
+    });
+
     it('leaves a suspension it did not cause alone', async () => {
         rows.push(
             durableRow({
@@ -283,7 +406,7 @@ describe('a suspension waiting on a balance', () => {
             }),
         );
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 });
@@ -298,9 +421,10 @@ describe('what a plan lets an account subscribe to', () => {
         await subscribe(appActor(`app-${seq}`));
 
         expect(created[0].limits).toEqual({
-            perUser: EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
-                DEFAULT_FREE_SUBSCRIPTION
-            ],
+            perUser:
+                EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
+                    DEFAULT_FREE_SUBSCRIPTION
+                ],
             perApp: EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP.bySubscription[
                 DEFAULT_FREE_SUBSCRIPTION
             ],

src/backend/services/events/registry.test.ts

@@ -28,6 +28,7 @@ import {
     lookupFsSubject,
     lookupKvSubject,
     lookupPublicSubject,
+    projectScheduleTick,
     type FsDeliveryContext,
     type FsPublicSubject,
     type KvDeliveryContext,
@@ -324,3 +325,52 @@ describe('registry typing', () => {
         expect(subjects.length).toBeGreaterThan(0);
     });
 });
+
+describe('the schedule tick', () => {
+    const tick = {
+        scheduleId: '2f7c1b6e-4d1a-4c55-9a8e-0e6f3c2b1a90',
+        scheduledFor: 1_798_761_600_000,
+        missed: 0,
+        missedCapped: false,
+        attempt: 1,
+    };
+
+    it('carries exactly the fields a handler is promised', () => {
+        const event = projectScheduleTick(tick);
+        expect(Object.keys(event).sort()).toEqual([
+            'attempt',
+            'id',
+            'missed',
+            'op',
+            'scheduleId',
+            'scheduledFor',
+            'subject',
+        ]);
+        expect(event).toEqual({
+            subject: `schedule:${tick.scheduleId}`,
+            op: 'tick',
+            id: `${tick.scheduleId}:${tick.scheduledFor}`,
+            scheduleId: tick.scheduleId,
+            scheduledFor: tick.scheduledFor,
+            missed: 0,
+            attempt: 1,
+        });
+    });
+
+    it('says when `missed` stopped counting, and only then', () => {
+        const capped = projectScheduleTick({
+            ...tick,
+            missed: 1000,
+            missedCapped: true,
+        });
+        expect(capped.missedCapped).toBe(true);
+        expect(capped.missed).toBe(1000);
+        expect('missedCapped' in projectScheduleTick(tick)).toBe(false);
+    });
+
+    it('keeps the id across attempts at one slot', () => {
+        expect(projectScheduleTick({ ...tick, attempt: 3 }).id).toBe(
+            projectScheduleTick(tick).id,
+        );
+    });
+});

src/backend/services/events/registry.ts

@@ -20,6 +20,7 @@
 import type { EventKey, KvOp } from '../../clients/event/types.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
 import type { AclMode } from '../acl/ACLService.js';
+import { scheduleToken } from '../../stores/events/types.js';
 import type { NotificationAudience } from '../notification/notificationTypes.js';
 import { relativeTo } from './matcher.js';
 import {
@@ -92,24 +93,25 @@ export interface ProjectedKvEvent extends ProjectedEventBase {
 /**
  * What a notification subscriber receives.
  *
- * `id` is the row's own uid rather than a per-dispatch uuid: a client that
- * fetched a missed notification and then received the live copy has to be able
- * to tell they are the same thing, and the mailbox uid is the only identity
- * both halves share. `uid` carries it under the name the mailbox verbs
- * (`mark-ack`, `mark-shown`) take.
+ * For postings, `id` is the row's own uid: a client that fetched a missed
+ * notification and then received the live copy has to be able to tell they are
+ * the same thing, and the mailbox uid is the only identity both halves share.
+ * `uid` carries it under the name the mailbox verbs (`mark-ack`, `mark-shown`)
+ * take.
  *
  * `notification` is the payload the caller sent, which is exactly what the
  * socket adapter puts on `notif.message` — a subscriber reading this channel
- * sees no less than the desktop does, and no internal row either.
+ * sees no less than the desktop does, and no internal row either. Dismissals
+ * use a fresh dispatch id and omit the payload.
  */
 export interface ProjectedNotifEvent extends ProjectedEventBase {
-    op: 'post';
+    op: 'post' | 'dismiss';
     uid: string;
     type: string;
     audience: NotificationAudience;
     /** App the notification is about; `null` for platform notifications. */
     appUid: string | null;
-    notification: Record<string, unknown>;
+    notification?: Record<string, unknown>;
     /**
      * Present when the notification replaced an earlier one under the same uid.
      * `ts` is then the replacement time, the same on live and fetched copies,
@@ -199,6 +201,7 @@ export interface KvEventContext extends EventContextBase {
  * recipient's uuid because the anchor token is the mailbox.
  */
 export interface NotifEventContext extends EventContextBase {
+    dismissed?: true;
     userId: number;
     userUuid: string;
     uid: string;
@@ -345,20 +348,64 @@ const notifMatchTarget = (event: NotifEventContext): string =>
 const notifMatchScope = (_anchorPath: string, slice: string): string => slice;
 
 const notifProject = (delivery: NotifDeliveryContext): ProjectedNotifEvent => ({
-    id: delivery.uid,
+    id: delivery.dismissed ? delivery.id : delivery.uid,
     subject: `notif:${delivery.appUid ?? delivery.userUuid}:${delivery.audience}`,
-    op: 'post',
+    op: delivery.dismissed ? 'dismiss' : 'post',
     uid: delivery.uid,
     type: delivery.type,
     audience: delivery.audience,
     appUid: delivery.appUid,
-    notification: delivery.notification,
+    ...(delivery.dismissed ? {} : { notification: delivery.notification }),
     ...(delivery.replaced ? { replaced: true as const } : {}),
     self: delivery.self,
     ts: delivery.ts,
     seq: delivery.seq,
 });
 
+// -- Schedule ticks ---------------------------------------------------
+
+/**
+ * What a schedule's handler is handed as `event`. No bus event stands behind
+ * it, so it has no registry entry: the runner builds it from the slot it fires.
+ * Additive only, as for the other projections.
+ */
+export interface ScheduleEvent {
+    /** `schedule:<scheduleId>`. */
+    subject: string;
+    op: 'tick';
+    /** `${scheduleId}:${scheduledFor}`: the same across retries of one slot. */
+    id: string;
+    scheduleId: string;
+    /** The slot this run is for, in epoch ms. */
+    scheduledFor: number;
+    /** When the invocation was sent; stamped as it goes out. */
+    firedAt?: number;
+    /** Slots skipped since the last one that ran. */
+    missed: number;
+    /** Set when `missed` stopped counting at its cap. */
+    missedCapped?: true;
+    /** Attempts at this slot, this one included. */
+    attempt: number;
+}
+
+/** The {@link ScheduleEvent} of one run. Called by extensions. */
+export const projectScheduleTick = (tick: {
+    scheduleId: string;
+    scheduledFor: number;
+    missed: number;
+    missedCapped: boolean;
+    attempt: number;
+}): ScheduleEvent => ({
+    subject: scheduleToken(tick.scheduleId),
+    op: 'tick',
+    id: `${tick.scheduleId}:${tick.scheduledFor}`,
+    scheduleId: tick.scheduleId,
+    scheduledFor: tick.scheduledFor,
+    missed: tick.missed,
+    ...(tick.missedCapped ? { missedCapped: true as const } : {}),
+    attempt: tick.attempt,
+});
+
 // -- Registry ---------------------------------------------------------
 
 export const PUBLIC_SUBJECTS = [
@@ -419,7 +466,7 @@ export const PUBLIC_SUBJECTS = [
     {
         family: 'notif',
         subject: 'notif:*',
-        internal: ['notif.created'],
+        internal: ['notif.created', 'notif.acknowledged'],
         tokens: notifTokens,
         matchOn: notifMatchTarget,
         matchSeparator: NOTIF_MATCH_SEPARATOR,

src/backend/services/events/requestParsing.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { HttpError } from '../../core/http/HttpError.js';
+import { checkRateLimit } from '../../core/http/middleware/rateLimit.js';
+import type { RouteRateLimit } from '../../core/http/types.js';
+import {
+    HANDLER_NAME_MAX_LENGTH,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
+import type { HandlerScopeKind } from '../../stores/events/types.js';
+
+/** Request and response pieces the subscription and schedule verbs share. */
+
+export const badRequest = (message: string, code: string): HttpError =>
+    new HttpError(400, message, { legacyCode: code });
+
+export const parseHandlerName = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || value.length === 0)
+        throw badRequest('handlerName must be a string', 'bad_request');
+    if (value.length > HANDLER_NAME_MAX_LENGTH)
+        throw badRequest(
+            `handlerName may not exceed ${HANDLER_NAME_MAX_LENGTH} characters`,
+            'bad_request',
+        );
+    return value;
+};
+
+/** Hex digest of the inline source a subscribe claims it is binding. */
+export const parseHandlerHash = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value))
+        throw badRequest(
+            'handlerHash must be a sha-256 hex digest',
+            'bad_request',
+        );
+    return value;
+};
+
+/** Source of an inline handler; the store enforces the size and content rules. */
+export const parseHandlerSource = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || value.trim().length === 0)
+        throw badRequest(
+            'handlerSource must be a non-empty string',
+            'bad_request',
+        );
+    return value;
+};
+
+/** Which handler set a request is about; absent means the app's. */
+export const parseHandlerScopeKind = (
+    value: unknown,
+    field: string,
+): HandlerScopeKind => {
+    if (value === undefined || value === null || value === 'app') return 'app';
+    if (value === 'user') return 'user';
+    throw badRequest(`${field} must be 'app' or 'user'`, 'bad_request');
+};
+
+/** Stored as JSON text; the byte cap is the store's to enforce. */
+export const parseContext = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    try {
+        return JSON.stringify(value);
+    } catch {
+        throw badRequest('context must be JSON-serializable', 'bad_request');
+    }
+};
+
+const tooManyCalls = (): HttpError =>
+    new HttpError(429, 'Too many subscription changes', {
+        legacyCode: 'too_many_requests',
+    });
+
+/** Spend one call from a per-user window, refusing past it. */
+export const spendUserWindow = async (
+    limit: RouteRateLimit,
+    userId: number,
+): Promise<void> => {
+    const ok = await checkRateLimit(
+        `${limit.scope}:${userId}`,
+        limit.limit,
+        limit.window,
+    );
+    if (!ok) throw tooManyCalls();
+};
+
+/**
+ * A stored context as a listing may describe it: which keys it sets, and a hash
+ * of the whole blob. Never the values: the column is where an API key lives.
+ */
+export const projectContext = (
+    context: string | null,
+): { contextKeys: string[] | null; contextHash: string | null } => {
+    if (context === null) return { contextKeys: null, contextHash: null };
+    let keys: string[] = [];
+    try {
+        const parsed: unknown = JSON.parse(context);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
+            keys = Object.keys(parsed).sort();
+    } catch {
+        // Stored as JSON by the verbs, so this cannot normally happen; an
+        // unreadable blob still reports its hash rather than failing the list.
+    }
+    return { contextKeys: keys, contextHash: hashContent(context) };
+};

src/backend/services/events/revocationSettle.integration.test.ts

@@ -27,7 +27,15 @@
  * out of service so a revoked subscription stops costing anything at all.
  */
 
-import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import {
+    afterAll,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    onTestFinished,
+    vi,
+} from 'vitest';
 import { v4 as uuidv4 } from 'uuid';
 import { EVENTS_COALESCE_WINDOW_MS } from '../../controllers/events/limits.js';
 import { SUSPENDED_ROW_TTL_DAYS } from '../../controllers/events/limits.js';
@@ -43,6 +51,7 @@ import type { AclMode } from '../acl/ACLService.js';
 import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import type { DeliveryEnvelope } from './EventsService.js';
 import { fsAnchorToken } from './subjects.js';
+import { appHandlerScope } from '../../stores/events/EventHandlerStore.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 const TABLE = 'event_subscriptions';
@@ -206,7 +215,7 @@ const makeApp = async (path: string): Promise<Actor> => {
         EVENTS_BACKGROUND_PERMISSION,
     );
     await env.server.stores.eventHandler.publish({
-        appUid: uid,
+        scope: appHandlerScope(uid),
         name: 'onChange',
         source: 'async () => {}',
     });
@@ -950,3 +959,165 @@ describe('what a revoked grant settles', () => {
         }
     });
 });
+
+describe('a holder`s user scope when consent is withdrawn', () => {
+    const scopeOf = (appUid: string, holder = owner.id) => ({
+        appUid,
+        holder,
+    });
+    const putUserHandler = async (appUid: string, holder = owner.id) => {
+        await env.server.stores.eventHandler.publish({
+            scope: scopeOf(appUid, holder),
+            name: 'mine',
+            source: 'async () => {}',
+            broadcast: false,
+        });
+    };
+    const held = (appUid: string, holder = owner.id) =>
+        env.server.stores.eventHandler.listForApp(scopeOf(appUid, holder));
+
+    let destroys: Array<{ appUid: string; scope: string; userId: number }>;
+    beforeAll(() => {
+        env.server.clients.event.on('events.worker.destroy', (_key, event) => {
+            destroys?.push({
+                appUid: event.appUid,
+                scope: event.scope,
+                userId: event.actor.user.id,
+            });
+        });
+    });
+
+    it('drops the scope when background consent is revoked, announcing it as the holder', async () => {
+        await clearRows();
+        destroys = [];
+        const path = await folder(`/${owner.username}/scope-consent`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+
+        await env.server.services.permission.revokeUserAppPermission(
+            owner.actor,
+            app.uid,
+            EVENTS_BACKGROUND_PERMISSION,
+        );
+
+        await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
+            timeout: 5_000,
+            interval: 25,
+        });
+        expect(destroys.filter((event) => event.scope === 'user')).toEqual([
+            { appUid: app.uid, scope: 'user', userId: owner.id },
+        ]);
+    });
+
+    it('drops the scope when the whole app is revoked', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-whole-app`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+
+        await env.server.services.permission.revokeUserAppAll(
+            owner.actor,
+            app.uid,
+        );
+
+        await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
+            timeout: 5_000,
+            interval: 25,
+        });
+    });
+
+    it('leaves the scope when the consent still stands, as after a revoke then a grant', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-regrant`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+
+        for (const permission of [EVENTS_BACKGROUND_PERMISSION, null])
+            await events().settleRevokedGrant({
+                holderUserId: owner.id,
+                appUid: app.uid,
+                permission,
+            });
+
+        expect(await held(app.uid)).toHaveLength(1);
+    });
+
+    it('leaves the scope when the consent check itself fails', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-check-fails`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+        const check = vi
+            .spyOn(events(), 'hasBackgroundConsent')
+            .mockResolvedValue(null);
+        try {
+            await events().settleRevokedGrant({
+                holderUserId: owner.id,
+                appUid: app.uid,
+                permission: EVENTS_BACKGROUND_PERMISSION,
+            });
+            expect(check).toHaveBeenCalled();
+            expect(await held(app.uid)).toHaveLength(1);
+        } finally {
+            check.mockRestore();
+        }
+    });
+
+    it('leaves the scope for an unrelated grant revoked, and another holder`s scope for any', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-unrelated`);
+        const app = (await makeApp(path)).app!;
+        await putUserHandler(app.uid);
+        await putUserHandler(app.uid, guest.id);
+
+        await env.server.services.permission.revokeUserAppPermission(
+            owner.actor,
+            app.uid,
+            `fs:${await uidOf(path)}:list`,
+        );
+        await env.server.services.permission.revokeUserAppPermission(
+            owner.actor,
+            app.uid,
+            EVENTS_BACKGROUND_PERMISSION,
+        );
+
+        await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
+            timeout: 5_000,
+            interval: 25,
+        });
+        expect(await held(app.uid, guest.id)).toHaveLength(1);
+    });
+
+    it('keeps the rows bound to a dropped scope, suspended rather than deleted', async () => {
+        await clearRows();
+        const path = await folder(`/${owner.username}/scope-rows`);
+        const appActor = await makeApp(path);
+        const app = appActor.app!;
+        const config = (events() as unknown as { config: IConfig }).config;
+        config.events!.userHandlers = true;
+        onTestFinished(() => {
+            config.events!.userHandlers = false;
+        });
+        const sub = (
+            await events().subscribeDurable(appActor, {
+                subject: `fs:${path}`,
+                handlerSource: 'async () => {}',
+            })
+        ).sub;
+
+        await env.server.services.permission.revokeUserAppPermission(
+            owner.actor,
+            app.uid,
+            EVENTS_BACKGROUND_PERMISSION,
+        );
+
+        await suspendedRow(sub.subId);
+        await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
+            timeout: 5_000,
+            interval: 25,
+        });
+        expect((await rowOf(sub.subId)).suspended_reason).toBe(
+            'permission_revoked',
+        );
+    });
+});

src/backend/services/events/scheduleSeam.integration.test.ts

@@ -0,0 +1,345 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The subscription lifecycle hands schedules' rows, and only theirs, to
+ * `services.schedules` with the change each path made. Without it every path
+ * still runs.
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type {
+    ScheduleServiceContract,
+    ScheduleSubscriptionChange,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import type { DurableSubscriptionInput } from '../../stores/events/DurableSubscriptionStore.js';
+import {
+    isScheduleRow,
+    scheduleToken,
+    type DurableSubscription,
+} from '../../stores/events/types.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
+
+const BOOT_TIMEOUT_MS = 120_000;
+const APP_UID = 'app-seam';
+
+let server: PuterServer;
+let userId: number;
+let anchorUid: string;
+let anchorPath: string;
+let saved: unknown;
+
+const durable = () => server.stores.durableSubscription;
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+
+const recording = () => {
+    const calls: Array<{
+        subIds: string[];
+        change: ScheduleSubscriptionChange;
+    }> = [];
+    const schedules = {
+        onSubscriptionsChanged: vi.fn(
+            async (
+                rows: readonly DurableSubscription[],
+                change: ScheduleSubscriptionChange,
+            ) => {
+                calls.push({ subIds: rows.map((r) => r.subId), change });
+            },
+        ),
+        createFor: vi.fn(),
+        list: vi.fn(),
+        get: vi.fn(),
+        update: vi.fn(),
+        remove: vi.fn(),
+        receivePeerChanges: vi.fn(),
+        prepareSendAt: vi.fn(),
+        createDeferredAppMessage: vi.fn(),
+    } satisfies ScheduleServiceContract;
+    setSchedules(schedules);
+    return calls;
+};
+
+const plainRow = async (
+    over: Partial<DurableSubscriptionInput> = {},
+): Promise<DurableSubscription> =>
+    (
+        await durable().create({
+            holderUserId: userId,
+            ownerUserId: userId,
+            appUid: APP_UID,
+            subject: `fs:${anchorPath}`,
+            token: `f#${anchorUid}`,
+            anchorUid,
+            anchorPath,
+            match: null,
+            op: null,
+            delivery: 'broadcast',
+            handlerName: 'onWrite',
+            context: null,
+            permission: 'list',
+            expiresAt: null,
+            ...over,
+        })
+    ).row;
+
+const scheduleRow = async (
+    over: Partial<DurableSubscriptionInput> = {},
+): Promise<DurableSubscription> => {
+    const uid = uuidv4();
+    return plainRow({
+        subject: scheduleToken(uid),
+        token: scheduleToken(uid),
+        anchorUid: uid,
+        anchorPath: '',
+        delivery: 'single',
+        handlerName: 'tick',
+        permission: 'events:background',
+        ...over,
+    });
+};
+
+const pendingCalls = () => {
+    const pending = server.stores.pendingDelivery;
+    return [
+        vi.spyOn(pending, 'purge'),
+        vi.spyOn(pending, 'hold'),
+        vi.spyOn(pending, 'releaseHold'),
+    ];
+};
+
+beforeAll(async () => {
+    server = await setupTestServer({ events: { enabled: true } } as IConfig);
+    const username = `seam-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+    });
+    userId = user.id;
+    anchorPath = `/${username}/seam`;
+    await server.services.fs.mkdir(userId, {
+        path: anchorPath,
+        createMissingParents: true,
+    });
+    anchorUid = (await server.stores.fsEntry.getEntryByPath(anchorPath))!.uid;
+    saved = slot().schedules;
+}, BOOT_TIMEOUT_MS);
+
+beforeEach(async () => {
+    await server.clients.db.write('DELETE FROM `event_subscriptions`', []);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+    vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+describe('the schedules lifecycle hook', () => {
+    it('gets only schedule rows on a suspend, and their backlog is never touched', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        const plain = await plainRow();
+        const [purge, hold, releaseHold] = pendingCalls();
+
+        await server.services.events.suspendSubscriptions(
+            [schedule],
+            'no_credit',
+        );
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'suspended', reason: 'no_credit' },
+            },
+        ]);
+        expect(purge).not.toHaveBeenCalled();
+        expect(hold).not.toHaveBeenCalled();
+
+        calls.length = 0;
+        await server.services.events.suspendSubscriptions([plain], 'no_credit');
+        expect(calls).toEqual([]);
+
+        calls.length = 0;
+        const suspended = (await durable().getBySubId(schedule.subId))!;
+        await server.services.events.resumeSubscriptions([suspended]);
+        expect(calls).toEqual([
+            { subIds: [schedule.subId], change: { type: 'resumed' } },
+        ]);
+        expect(releaseHold).not.toHaveBeenCalled();
+    });
+
+    it('ends schedules uninstalled when every grant to the app goes', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: null,
+        });
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: {
+                    type: 'suspended',
+                    reason: 'permission_revoked',
+                    endReason: 'uninstalled',
+                },
+            },
+        ]);
+    });
+
+    it('reads no schedule rows to settle a specific grant, and still settles them when consent goes', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        const plain = await plainRow();
+        const page = vi.spyOn(durable(), 'listActiveForHolderPage');
+        const readSubIds = async () =>
+            (await Promise.all(page.mock.results.map((r) => r.value))).flatMap(
+                (p) => p.rows.map((row: DurableSubscription) => row.subId),
+            );
+
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: `fs:${anchorUid}:list`,
+        });
+        const read = await readSubIds();
+        expect(read).toContain(plain.subId);
+        expect(read).not.toContain(schedule.subId);
+        expect(calls).toEqual([]);
+
+        page.mockClear();
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: EVENTS_BACKGROUND_PERMISSION,
+        });
+        expect(await readSubIds()).toContain(schedule.subId);
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'suspended', reason: 'permission_revoked' },
+            },
+        ]);
+    });
+
+    it('tells the holder once per kind when an uninstall settles several pages', async () => {
+        recording();
+        const plains = [await plainRow(), await plainRow(), await plainRow()];
+        const schedules = [await scheduleRow(), await scheduleRow()];
+        const store = durable();
+        const listPage = store.listActiveForHolderPage.bind(store);
+        vi.spyOn(store, 'listActiveForHolderPage').mockImplementation(
+            (holder, app, afterId, _batchSize, opts) =>
+                listPage(holder, app, afterId, 1, opts),
+        );
+        const notify = vi.spyOn(server.services.notification, 'notify');
+
+        await expect(
+            server.services.events.settleRevokedGrant({
+                holderUserId: userId,
+                appUid: APP_UID,
+                permission: null,
+            }),
+        ).resolves.toBe(5);
+        const ended = notify.mock.calls
+            .filter(([, , meta]) => meta?.type === 'app.events.ended')
+            .map(([, body]) => body as { count: number; subjects: string[] });
+        expect(ended).toHaveLength(2);
+        expect(ended.map((body) => body.count).sort()).toEqual([2, 3]);
+        expect(ended.flatMap((body) => body.subjects).sort()).toEqual(
+            [...plains, ...schedules].map((row) => row.subject).sort(),
+        );
+    });
+
+    it('ends schedules app_deleted when their app is deleted', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await server.services.events.settleDeletedApp(APP_UID);
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'ended', endReason: 'app_deleted' },
+            },
+        ]);
+    });
+
+    it('ends schedules whose rows expire', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow({ expiresAt: 1 });
+        await plainRow({ expiresAt: 1 });
+
+        await server.services.events.sweepExpired();
+        const handed = calls.flatMap((c) => c.subIds);
+        expect(handed).toEqual([schedule.subId]);
+        expect(calls[0].change).toEqual({ type: 'ended', endReason: 'ended' });
+    });
+
+    it('lets every path run without services.schedules', async () => {
+        setSchedules(undefined);
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await expect(
+            server.services.events.suspendSubscriptions(
+                [schedule],
+                'no_credit',
+            ),
+        ).resolves.toBe(1);
+        await expect(
+            server.services.events.resumeSubscriptions([
+                (await durable().getBySubId(schedule.subId))!,
+            ]),
+        ).resolves.toBe(1);
+        await server.services.events.settleDeletedApp(APP_UID);
+        const left = await server.clients.db.read(
+            'SELECT `token` FROM `event_subscriptions`',
+            [],
+        );
+        expect(
+            left.filter((r) => isScheduleRow(r as { token: string })),
+        ).toEqual([]);
+    });
+});

src/backend/services/events/subjects.ts

@@ -48,8 +48,7 @@ export type SubjectFamily = 'fs' | 'kv' | 'notif';
 
 export type FsOp = 'add' | 'write' | 'move' | 'remove' | 'meta';
 
-/** A notification is only ever posted; the mailbox verbs are their own surface. */
-export type NotifOp = 'post';
+export type NotifOp = 'post' | 'dismiss';
 
 export type SubjectOp = FsOp | KvOp | NotifOp;
 

src/backend/services/events/userHandlers.worker.integration.test.ts

@@ -0,0 +1,386 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * A user-scope handler through the real local worker runtime: created inline by
+ * an app session, deployed eagerly as its holder, fired by an event, and run
+ * with the holder's identity. Also the rehydrate route's scope resolution.
+ */
+
+import http from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import {
+    appHandlerScope,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
+import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
+import { userHandlerScript } from './workerRuntime.js';
+
+const BOOT_TIMEOUT_MS = 180_000;
+const TEST_TIMEOUT_MS = 60_000;
+const INTERNAL_SECRET = 'events-internal-secret';
+
+const SOURCE = `async ({ event, ctx, user, fetch, ack }) => {
+    await fetch(ctx.sink, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({
+            label: ctx.label,
+            path: event.path,
+            token: user.authToken,
+        }),
+    });
+    await ack();
+}`;
+
+interface SinkPost {
+    label?: string;
+    path?: string;
+    token?: string;
+}
+
+let env: PuterTestEnv;
+let holders: Array<{ id: number; uuid: string; anchor: string; token: string }>;
+let appUid: string;
+let sink: http.Server;
+let sinkUrl: string;
+let sinkPosts: SinkPost[];
+
+const localWorkers = () => env.server.services.localworkerservice;
+
+const scriptFor = (source: string): string =>
+    userHandlerScript(
+        { name: `@${hashContent(source)}`, sourceHash: hashContent(source) },
+        '',
+    );
+
+const isResident = async (script: string): Promise<boolean> =>
+    (await localWorkers().dispatchEventsWorker(
+        new Request(`http://${script}.workers.puter.localhost/`, {
+            method: 'GET',
+        }),
+    )) !== null;
+
+const api = async (
+    method: 'GET' | 'POST',
+    path: string,
+    token: string,
+    body?: object,
+): Promise<{ status: number; body: Record<string, unknown> }> => {
+    const response = await fetch(new URL(path, env.apiOrigin), {
+        method,
+        headers: {
+            'content-type': 'application/json',
+            authorization: `Bearer ${token}`,
+        },
+        ...(body ? { body: JSON.stringify(body) } : {}),
+    });
+    return {
+        status: response.status,
+        body: (await response.json()) as Record<string, unknown>,
+    };
+};
+
+const rehydrate = async (
+    body: object,
+): Promise<{ status: number; body: Record<string, unknown> }> => {
+    const response = await fetch(
+        new URL('/events/worker/rehydrate', env.apiOrigin),
+        {
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json',
+                'x-puter-internal-auth': INTERNAL_SECRET,
+            },
+            body: JSON.stringify(body),
+        },
+    );
+    return {
+        status: response.status,
+        body: (await response.json()) as Record<string, unknown>,
+    };
+};
+
+const createInline = async (
+    holder: (typeof holders)[number],
+    label: string,
+    source = SOURCE,
+): Promise<string> => {
+    const response = await api('POST', '/events/subscribe', holder.token, {
+        subject: `fs:${holder.anchor}`,
+        delivery: 'single',
+        handlerSource: source,
+        context: { sink: sinkUrl, label },
+    });
+    expect(response.status).toBe(200);
+    return String(response.body.subId);
+};
+
+const delivered = (count = 1): Promise<void> =>
+    vi.waitFor(() => expect(sinkPosts.length).toBeGreaterThanOrEqual(count), {
+        timeout: 30_000,
+        interval: 50,
+    });
+
+beforeAll(async () => {
+    env = await setupPuterTestEnv({
+        events: {
+            enabled: true,
+            workerRuntime: true,
+            userHandlers: true,
+            internalSecret: INTERNAL_SECRET,
+        },
+        workers: { localServer: true },
+        unlimitedMetering: true,
+    } as IConfig);
+
+    const owner = (await env.server.stores.user.getByUsername(
+        env.users.user.username,
+    ))!;
+    appUid = `app-${uuidv4()}`;
+    await env.server.clients.db.write(
+        'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
+        [appUid, appUid, appUid, `https://${appUid}.example/`, owner.id],
+    );
+
+    holders = [];
+    for (const account of [env.users.user, env.users.other]) {
+        const user = (await env.server.stores.user.getByUsername(
+            account.username,
+        ))!;
+        const anchor = `/${account.username}/uh-e2e`;
+        await env.server.services.fs.mkdir(user.id, {
+            path: anchor,
+            createMissingParents: true,
+        });
+        const entry = await env.server.stores.fsEntry.getEntryByPath(anchor);
+        const { actor } = await env.server.services.auth.authenticate(
+            account.token,
+        );
+        for (const permission of [
+            `fs:${entry!.uid}:list`,
+            EVENTS_BACKGROUND_PERMISSION,
+        ])
+            await env.server.services.permission.grantUserAppPermission(
+                actor!,
+                appUid,
+                permission,
+            );
+        holders.push({
+            id: user.id,
+            uuid: user.uuid,
+            anchor,
+            token: await env.server.services.auth.getUserAppToken(
+                actor!,
+                appUid,
+            ),
+        });
+    }
+
+    sinkPosts = [];
+    sink = http.createServer((req, res) => {
+        const chunks: Buffer[] = [];
+        req.on('data', (chunk: Buffer) => chunks.push(chunk));
+        req.on('end', () => {
+            sinkPosts.push(
+                JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'),
+            );
+            res.writeHead(200).end();
+        });
+    });
+    await new Promise<void>((resolve) =>
+        sink.listen(0, '127.0.0.1', () => resolve()),
+    );
+    sinkUrl = `http://127.0.0.1:${(sink.address() as AddressInfo).port}`;
+}, BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+    await new Promise<void>((resolve) => {
+        if (!sink) return resolve();
+        sink.closeAllConnections();
+        sink.close(() => resolve());
+    });
+    await env?.shutdown();
+});
+
+beforeEach(() => {
+    sinkPosts = [];
+});
+
+describe('an inline handler in a user scope, end to end', () => {
+    it(
+        'is deployed eagerly, fires on an event and runs as its holder',
+        { timeout: TEST_TIMEOUT_MS },
+        async () => {
+            const [holder] = holders;
+            const source = `${SOURCE}\n// run-as-holder`;
+            const script = scriptFor(source);
+            expect(await isResident(script)).toBe(false);
+
+            await createInline(holder, 'inline', source);
+
+            // The deploy started with the publish, before any event.
+            await vi.waitFor(async () =>
+                expect(await isResident(script)).toBe(true),
+            );
+
+            await env.server.services.fs.touch(holder.id, {
+                path: `${holder.anchor}/note.txt`,
+            });
+            await delivered();
+
+            expect(sinkPosts[0]).toMatchObject({
+                label: 'inline',
+                path: `${holder.anchor}/note.txt`,
+            });
+            const { actor } = await env.server.services.auth.authenticate(
+                sinkPosts[0].token!,
+            );
+            expect(actor!.user?.id).toBe(holder.id);
+            expect(actor!.effectiveApp?.uid).toBe(appUid);
+        },
+    );
+
+    it(
+        'shares one deployed script between holders of the same code, each run as its own holder',
+        { timeout: TEST_TIMEOUT_MS },
+        async () => {
+            const source = `${SOURCE}\n// shared-code`;
+            const script = scriptFor(source);
+            const create = vi.spyOn(env.server.drivers.workers, 'create');
+
+            await createInline(holders[0], 'first', source);
+            await vi.waitFor(async () =>
+                expect(await isResident(script)).toBe(true),
+            );
+            await createInline(holders[1], 'second', source);
+            // Let the second holder's eager deploy settle before counting.
+            await new Promise((resolve) => setTimeout(resolve, 500));
+
+            // The marker made the second holder's eager deploy a no-op.
+            expect(
+                create.mock.calls.filter(
+                    ([args]) =>
+                        (args as { workerName: string }).workerName === script,
+                ),
+            ).toHaveLength(1);
+
+            for (const [i, holder] of holders.entries())
+                await env.server.services.fs.touch(holder.id, {
+                    path: `${holder.anchor}/shared-${i}.txt`,
+                });
+            await delivered(2);
+            const runAs = new Set<number>();
+            for (const post of sinkPosts)
+                runAs.add(
+                    (await env.server.services.auth.authenticate(post.token!))
+                        .actor!.user!.id,
+                );
+            expect(runAs).toEqual(new Set(holders.map((h) => h.id)));
+            create.mockRestore();
+        },
+    );
+});
+
+describe('the rehydrate route', () => {
+    const source = `${SOURCE}\n// rehydrate`;
+    const script = scriptFor(source);
+
+    beforeAll(async () => {
+        // Rows only: the handler is published, nothing is deployed here.
+        const eventHandler = env.server.stores.eventHandler;
+        await eventHandler.publish({
+            scope: { appUid, holder: holders[0].id },
+            name: `@${hashContent(source)}`,
+            source,
+            broadcast: false,
+        });
+        await eventHandler.publish({
+            scope: { appUid, holder: holders[1].id },
+            name: 'other',
+            source: SOURCE,
+            broadcast: false,
+        });
+    });
+
+    it('resolves the holder`s user scope when the body names one', async () => {
+        const response = await rehydrate({
+            script,
+            appUid,
+            holder: holders[0].uuid,
+        });
+        expect(response).toMatchObject({
+            status: 200,
+            body: { deployed: true },
+        });
+        expect(await isResident(script)).toBe(true);
+    });
+
+    it('reads an absent or empty holder as the app scope, which has no such handler', async () => {
+        for (const body of [
+            { script, appUid },
+            { script, appUid, holder: '' },
+        ])
+            expect(await rehydrate(body)).toMatchObject({
+                status: 404,
+                body: { deployed: false, reason: 'no-handlers' },
+            });
+        expect(
+            await env.server.stores.eventHandler.listForApp(
+                appHandlerScope(appUid),
+            ),
+        ).toEqual([]);
+    });
+
+    it('answers stale for a holder whose scope does not hold that handler', async () => {
+        expect(
+            await rehydrate({ script, appUid, holder: holders[1].uuid }),
+        ).toMatchObject({
+            status: 404,
+            body: { deployed: false, reason: 'stale' },
+        });
+    });
+
+    it('refuses a holder that does not exist, and a malformed one', async () => {
+        expect(
+            await rehydrate({ script, appUid, holder: uuidv4() }),
+        ).toMatchObject({
+            status: 404,
+            body: { deployed: false, reason: 'no-holder' },
+        });
+        expect(
+            (await rehydrate({ script, appUid, holder: 'a b' })).status,
+        ).toBe(400);
+        expect((await rehydrate({ script, appUid, holder: 7 })).status).toBe(
+            400,
+        );
+    });
+});

src/backend/services/events/workerInvoker.integration.test.ts

@@ -34,6 +34,7 @@ import type { AddressInfo } from 'node:net';
 import { v4 as uuidv4 } from 'uuid';
 import {
     afterAll,
+    afterEach,
     beforeAll,
     beforeEach,
     describe,
@@ -58,8 +59,14 @@ import {
     EVENTS_WORKER_SESSION_NAME,
     eventsInvokeKey,
     eventsWorkerScript,
+    userHandlerScript,
 } from './workerRuntime.js';
 import { handlerSetHash } from './workerSource.js';
+import {
+    appHandlerScope,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
+import type { WorkerInvocation } from './workerSeam.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 /** Short enough that a hung handler is a test case rather than a stall. */
@@ -129,6 +136,7 @@ const startStub = async (): Promise<string> => {
                     script: req.headers['x-puter-events-script'] as string,
                     app: req.headers['x-puter-events-app'] as string,
                     key: req.headers['x-puter-events-key'] as string,
+                    holder: req.headers['x-puter-events-holder'] as string,
                     deployed,
                 },
                 body: JSON.parse(raw || '{}') as StubCall['body'],
@@ -233,7 +241,7 @@ const makeWorkerApp = async (): Promise<{
     );
     const token = await env.server.services.auth.getUserAppToken(actor!, uid);
     await env.server.stores.eventHandler.publish({
-        appUid: uid,
+        scope: appHandlerScope(uid),
         name: HANDLER,
         source: SOURCE,
     });
@@ -403,7 +411,7 @@ beforeAll(async () => {
     );
     appToken = await env.server.services.auth.getUserAppToken(actor!, appUid);
     await env.server.stores.eventHandler.publish({
-        appUid,
+        scope: appHandlerScope(appUid),
         name: HANDLER,
         source: SOURCE,
     });
@@ -471,7 +479,9 @@ describe('the call an owed delivery makes', () => {
 
         const script = eventsWorkerScript(
             handlerSetHash(
-                await env.server.stores.eventHandler.setForApp(appUid),
+                await env.server.stores.eventHandler.setForApp(
+                    appHandlerScope(appUid),
+                ),
             ),
             // No `workers.internetExposedUrl` in this test's config.
             '',
@@ -1014,7 +1024,10 @@ describe('with no events worker to address', () => {
         const subId = await subscribe();
         // Unpublished after the fact: the app's set now hashes to nothing, so
         // there is no script to name and nowhere to send the delivery.
-        await env.server.stores.eventHandler.remove(appUid, HANDLER);
+        await env.server.stores.eventHandler.remove(
+            appHandlerScope(appUid),
+            HANDLER,
+        );
 
         try {
             await touch('unresolved.txt');
@@ -1037,7 +1050,7 @@ describe('with no events worker to address', () => {
             expect(calls).toEqual([]);
         } finally {
             await env.server.stores.eventHandler.publish({
-                appUid,
+                scope: appHandlerScope(appUid),
                 name: HANDLER,
                 source: SOURCE,
             });
@@ -1416,3 +1429,349 @@ describe('handlers whose writes run handlers', () => {
         ).resolves.toBeNull();
     });
 });
+
+describe('a handler in a user scope', () => {
+    const USER_SOURCE =
+        'async ({ event, ctx }) => { console.log("user", event.path, ctx.label); }';
+    const userName = `@${hashContent(USER_SOURCE)}`;
+    let holderUuid: string;
+    let createSpy: ReturnType<typeof vi.spyOn>;
+    let meterSpy: ReturnType<typeof vi.spyOn>;
+
+    const setFlag = (on: boolean): void => {
+        (
+            events() as unknown as { config: IConfig }
+        ).config.events!.userHandlers = on;
+    };
+    const userScope = () => ({ appUid, holder: userId });
+    const userScript = () =>
+        userHandlerScript(
+            { name: userName, sourceHash: hashContent(USER_SOURCE) },
+            '',
+        );
+
+    const post = async (path: string, body: object) => {
+        const response = await fetch(new URL(path, env.apiOrigin), {
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json',
+                authorization: `Bearer ${appToken}`,
+            },
+            body: JSON.stringify(body),
+        });
+        return {
+            status: response.status,
+            body: (await response.json()) as Record<string, unknown>,
+        };
+    };
+    /** A single-delivery row bound to a handler published by its own source. */
+    const inlineRow = async (source = USER_SOURCE): Promise<string> => {
+        const created = await post('/events/subscribe', {
+            subject: `fs:${anchor}`,
+            delivery: 'single',
+            handlerSource: source,
+            context: { label: 'user' },
+        });
+        expect(created.status).toBe(200);
+        return String(created.body.subId);
+    };
+    const invocationFor = (subId: string): WorkerInvocation => ({
+        subId,
+        holderUserId: userId,
+        appUid,
+        handlerName: userName,
+        handlerScope: 'user',
+        event: {
+            subject: 'schedule:s1',
+            op: 'tick',
+            id: 's1:1',
+            scheduleId: 's1',
+            scheduledFor: 1,
+            missed: 0,
+            attempt: 1,
+        },
+        context: null,
+        handlerDepth: 0,
+    });
+
+    beforeAll(async () => {
+        holderUuid = (await env.server.stores.user.getById(userId))!.uuid;
+    });
+    beforeEach(async () => {
+        setFlag(true);
+        await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
+        // Eager and on-miss deploys have no deploy target in this suite.
+        createSpy = vi
+            .spyOn(env.server.drivers.workers, 'create')
+            .mockResolvedValue({ success: true, errors: [] });
+        meterSpy = vi.spyOn(
+            env.server.services.metering,
+            'bufferIncrementUsages',
+        );
+    });
+    afterEach(async () => {
+        createSpy.mockRestore();
+        meterSpy.mockRestore();
+        setFlag(false);
+        await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
+    });
+
+    it('addresses the handler`s own script and names its holder', async () => {
+        const subId = await inlineRow();
+        await touch('user-scope.txt');
+        await invoked(1);
+
+        const call = calls[0];
+        expect(call.headers.script).toBe(userScript());
+        expect(call.headers.holder).toBe(holderUuid);
+        expect(call.headers.key).toBe(
+            eventsInvokeKey(INTERNAL_SECRET, userScript()),
+        );
+        expect(call.body.handler).toBe(userName);
+        await waitUntil(async () =>
+            expect(await pending().depth(subId)).toBe(0),
+        );
+    });
+
+    it('sends no holder for an app-scope row', async () => {
+        await subscribe();
+        await touch('app-scope.txt');
+        await invoked(1);
+        expect(calls[0].headers.holder).toBeUndefined();
+    });
+
+    it('keeps a user row and an app row of the same name on their own scope`s handler', async () => {
+        await env.server.stores.eventHandler.publish({
+            scope: userScope(),
+            name: HANDLER,
+            source: USER_SOURCE,
+        });
+        const created = await post('/events/subscribe', {
+            subject: `fs:${anchor}`,
+            delivery: 'single',
+            handlerName: HANDLER,
+            handlerScope: 'user',
+        });
+        expect(created.status).toBe(200);
+        await touch('same-name-user.txt');
+        await invoked(1);
+        // One at a time: a second row firing alongside would go as a batch.
+        expect(
+            (
+                await post('/events/unsubscribe', {
+                    subId: String(created.body.subId),
+                })
+            ).status,
+        ).toBe(200);
+        await subscribe();
+        await touch('same-name-app.txt');
+        await invoked(2);
+
+        const appScript = eventsWorkerScript(
+            handlerSetHash(
+                await env.server.stores.eventHandler.setForApp(
+                    appHandlerScope(appUid),
+                ),
+            ),
+            '',
+        );
+        const mine = userHandlerScript(
+            { name: HANDLER, sourceHash: hashContent(USER_SOURCE) },
+            '',
+        );
+        expect(mine).not.toBe(appScript);
+        expect(calls[0].headers).toMatchObject({
+            script: mine,
+            holder: holderUuid,
+        });
+        expect(calls[1].headers.script).toBe(appScript);
+        expect(calls[1].headers.holder).toBeUndefined();
+    });
+
+    it('suspends a row whose handler left its holder`s set, and tells the holder only', async () => {
+        const notify = vi.spyOn(env.server.services.notification, 'notify');
+        const subId = await inlineRow();
+        // Gone without the removal path that would suspend its rows.
+        await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
+
+        await touch('missing.txt');
+        await waitUntil(async () =>
+            expect((await durable().getBySubId(subId))?.suspendedReason).toBe(
+                'handler_not_found',
+            ),
+        );
+        expect(calls).toEqual([]);
+        expect(notify).toHaveBeenCalledWith(
+            [userId],
+            expect.objectContaining({ handler: userName }),
+            expect.objectContaining({
+                type: 'app.events.handler_suspended',
+                appUid,
+            }),
+        );
+        expect(notify).not.toHaveBeenCalledWith(
+            expect.anything(),
+            expect.anything(),
+            expect.objectContaining({ type: 'app.events.suspended' }),
+        );
+        notify.mockRestore();
+    });
+
+    it('does not suspend a schedule run whose replica read lags the publish', async () => {
+        const subId = await inlineRow();
+        const store = env.server.stores.eventHandler;
+        const real = store.setForApp.bind(store);
+        // The replica has not seen the insert; the primary has.
+        const lagging = vi
+            .spyOn(store, 'setForApp')
+            .mockImplementation(async (scope, options) =>
+                options?.replica
+                    ? [{ name: 'other', sourceHash: 'a'.repeat(64) }]
+                    : real(scope, options),
+            );
+        try {
+            const result = await events().invokeForSchedule(
+                invocationFor(subId),
+            );
+            expect(result).toMatchObject({ outcome: 'settled' });
+            expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+            expect(calls[0].headers.script).toBe(userScript());
+        } finally {
+            lagging.mockRestore();
+        }
+    });
+
+    it('reads a cold user scope from the replica only, once, for a delivery', async () => {
+        await inlineRow();
+        // Eager deploy at publish has finished with the primary by now.
+        await new Promise((resolve) => setTimeout(resolve, 300));
+        const redis = env.server.clients.redis;
+        const key = `eventHandlers:uset:${appUid}:${userId}`;
+        await redis.del(key, `${key}:r`);
+        const db = env.server.clients.db;
+        const primary = vi.spyOn(db, 'pread');
+        const replica = vi.spyOn(db, 'read');
+        const handlerReads = (spy: typeof primary) =>
+            spy.mock.calls.filter(([sql]) =>
+                String(sql).includes('event_user_handlers'),
+            ).length;
+
+        await touch('cold-user-scope.txt');
+        await invoked(1);
+
+        expect(handlerReads(primary)).toBe(0);
+        expect(handlerReads(replica)).toBe(1);
+        primary.mockRestore();
+        replica.mockRestore();
+    });
+
+    it('resumes a suspended row when the name is published again', async () => {
+        const subId = await inlineRow();
+        await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
+        await touch('gone.txt');
+        await waitUntil(async () =>
+            expect((await durable().getBySubId(subId))?.suspendedReason).toBe(
+                'handler_not_found',
+            ),
+        );
+
+        expect(
+            (
+                await post('/events/handlers/publish', {
+                    scope: 'user',
+                    name: userName,
+                    source: USER_SOURCE,
+                })
+            ).status,
+        ).toBe(200);
+        expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+    });
+
+    it('does not address a holder that is suspended, and leaves the row owed', async () => {
+        const subId = await inlineRow();
+        await env.server.clients.db.write(
+            'UPDATE user SET suspended = 1 WHERE id = ?',
+            [userId],
+        );
+        await env.server.stores.user.invalidateById(userId);
+        try {
+            await touch('suspended-holder.txt');
+            await failuresRecorded(subId, 1);
+            expect(calls).toEqual([]);
+            expect(await pending().depth(subId)).toBe(1);
+            expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+        } finally {
+            await env.server.clients.db.write(
+                'UPDATE user SET suspended = 0 WHERE id = ?',
+                [userId],
+            );
+            await env.server.stores.user.invalidateById(userId);
+        }
+    });
+
+    it('deploys the holder`s script on a dispatcher miss, as that holder', async () => {
+        dispatchMissing = true;
+        await inlineRow();
+        await touch('miss.txt');
+        await invoked(2);
+
+        expect(calls[0].headers.deployed).toBeUndefined();
+        expect(calls[1].headers.deployed).toBe('1');
+        // Once for the eager deploy at publish, once for the miss.
+        const deployed = createSpy.mock.calls.filter(
+            ([args]) =>
+                (args as { workerName: string }).workerName === userScript(),
+        );
+        expect(deployed.length).toBeGreaterThanOrEqual(1);
+    });
+
+    describe('with the flag off', () => {
+        it('leaves the delivery owed: nothing is called, suspended or billed', async () => {
+            const subId = await inlineRow();
+            setFlag(false);
+            meterSpy.mockClear();
+
+            await touch('flag-off.txt');
+            await new Promise((resolve) => setTimeout(resolve, 400));
+
+            expect(calls).toEqual([]);
+            expect(await pending().depth(subId)).toBe(1);
+            expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+            expect(meterSpy).not.toHaveBeenCalled();
+        });
+
+        it('answers unavailable from the invoker and for a schedule run', async () => {
+            const subId = await inlineRow();
+            setFlag(false);
+            const invocation = invocationFor(subId);
+
+            const worker = events().worker as unknown as {
+                run: (i: WorkerInvocation) => Promise<unknown>;
+            };
+            expect(await worker.run(invocation)).toEqual({
+                outcome: 'unavailable',
+            });
+            expect(await events().invokeForSchedule(invocation)).toEqual({
+                outcome: 'unavailable',
+            });
+            expect(calls).toEqual([]);
+            expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
+        });
+
+        it('leaves an app-scope row alone', async () => {
+            setFlag(false);
+            await subscribe();
+            await touch('app-row-flag-off.txt');
+            await invoked(1);
+            expect(calls[0].headers.script).toMatch(/^evw-/);
+        });
+    });
+
+    it('runs a schedule invocation of a user scope with its holder', async () => {
+        const subId = await inlineRow();
+        const result = await events().invokeForSchedule(invocationFor(subId));
+        expect(result).toMatchObject({ outcome: 'settled' });
+        expect(calls[0].headers.script).toBe(userScript());
+        expect(calls[0].headers.holder).toBe(holderUuid);
+    });
+});

src/backend/services/events/workerRuntime.integration.test.ts

@@ -61,6 +61,7 @@ import {
     eventsWorkerScript,
 } from './workerRuntime.js';
 import { handlerSetHash } from './workerSource.js';
+import { appHandlerScope } from '../../stores/events/EventHandlerStore.js';
 
 const BOOT_TIMEOUT_MS = 180_000;
 const TEST_TIMEOUT_MS = 60_000;
@@ -171,7 +172,7 @@ const localWorkers = () => env.server.services.localworkerservice;
 /** The script the app's published set currently deploys as. */
 const scriptName = async (): Promise<string> =>
     eventsWorkerScript(
-        handlerSetHash(await env.server.stores.eventHandler.setForApp(appUid)),
+        handlerSetHash(await env.server.stores.eventHandler.setForApp(appHandlerScope(appUid))),
         // No `workers.internetExposedUrl` in this test's config, so the scope
         // this backend binds is empty.
         '',
@@ -886,7 +887,7 @@ describe('the worker follows the handler set', () => {
             // An empty set names no script, so nothing can be deployed for
             // this app again until something is published.
             expect(
-                await env.server.stores.eventHandler.setForApp(appUid),
+                await env.server.stores.eventHandler.setForApp(appHandlerScope(appUid)),
             ).toEqual([]);
             expect((await durable().getBySubId(subId))?.suspendedReason).toBe(
                 'handler_not_found',

src/backend/services/events/workerRuntime.test.ts

@@ -23,7 +23,9 @@ import {
     eventsInvokeKey,
     eventsWorkerScope,
     eventsWorkerScript,
+    userHandlerScript,
 } from './workerRuntime.js';
+import { handlerSetHash } from './workerSource.js';
 
 const SET_HASH = 'a1'.repeat(32);
 const OTHER_HASH = 'b2'.repeat(32);
@@ -50,6 +52,37 @@ describe('eventsWorkerScript', () => {
     });
 });
 
+describe('userHandlerScript', () => {
+    const handler = { name: '@abc', sourceHash: SET_HASH };
+
+    it('is the script of the handler as a set of one', () => {
+        expect(userHandlerScript(handler, '')).toBe(
+            eventsWorkerScript(handlerSetHash([handler]), ''),
+        );
+    });
+
+    it('is the same for identical code whoever holds it, and differs by code and name', () => {
+        // Nothing but name and source hash goes in: two holders (or two apps)
+        // of the same handler deploy one script.
+        const again = { ...handler };
+        expect(userHandlerScript(again, 's')).toBe(
+            userHandlerScript(handler, 's'),
+        );
+        expect(
+            userHandlerScript({ ...handler, sourceHash: OTHER_HASH }, 's'),
+        ).not.toBe(userHandlerScript(handler, 's'));
+        expect(userHandlerScript({ ...handler, name: '@abd' }, 's')).not.toBe(
+            userHandlerScript(handler, 's'),
+        );
+    });
+
+    it('stays scoped to the backend', () => {
+        expect(userHandlerScript(handler, 'https://a.example')).not.toBe(
+            userHandlerScript(handler, 'https://b.example'),
+        );
+    });
+});
+
 describe('eventsWorkerScope', () => {
     it('reads the same field `#cfDeploy` binds as `puter_endpoint`, and falls back to empty', () => {
         expect(eventsWorkerScope({})).toBe('');

src/backend/services/events/workerRuntime.ts

@@ -18,6 +18,8 @@
  */
 
 import { createHash, createHmac } from 'node:crypto';
+import type { EventHandler } from '../../stores/events/EventHandlerStore.js';
+import { handlerSetHash } from './workerSource.js';
 
 /**
  * How an app's events worker is named and how it recognizes the platform.
@@ -71,6 +73,17 @@ export const eventsWorkerScript = (setHash: string, scope: string): string =>
         .digest('hex')
         .slice(0, 32)}`;
 
+/**
+ * The script one user-scope handler deploys as: the set-of-one name, built from
+ * the handler's name and source hash alone. The holder and the app are not part
+ * of it, so identical code under one name is one script however many users and
+ * apps publish it.
+ */
+export const userHandlerScript = (
+    handler: Pick<EventHandler, 'name' | 'sourceHash'>,
+    scope: string,
+): string => eventsWorkerScript(handlerSetHash([handler]), scope);
+
 /** Version prefix on a derived key, so a rotation is visible in a log line. */
 export const EVENTS_INVOKE_KEY_VERSION = 'k1';
 

src/backend/services/events/workerSeam.ts

@@ -17,8 +17,15 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import type { EventsWorkerInvokerClient } from '../../clients/events/EventsWorkerInvokerClient.js';
-import type { DeliverableEvent } from './registry.js';
+import type {
+    EventsWorkerInvokerClient,
+    WorkerInvokeOptions,
+} from '../../clients/events/EventsWorkerInvokerClient.js';
+import type { Actor } from '../../core/actor.js';
+import type { HandlerScopeKind } from '../../stores/events/types.js';
+import type { DeliverableEvent, ScheduleEvent } from './registry.js';
+
+export type { ScheduleEvent } from './registry.js';
 
 /**
  * Where a delivery leaves the event system for the app's own code.
@@ -36,11 +43,15 @@ export interface WorkerInvocation {
     /** The app whose handler runs, or null for an account-owned row. */
     appUid: string | null;
     handlerName: string | null;
-    event: DeliverableEvent;
+    /** Whose handler set `handlerName` is in: absent means the app's own. */
+    handlerScope?: HandlerScopeKind;
+    event: DeliverableEvent | ScheduleEvent;
     /** The subscription's stored context, delivered to the handler as `ctx`. */
     context: string | null;
     /** Handler runs behind `event`; the token this run is handed is one deeper. */
     handlerDepth: number;
+    /** The holder acting as the app, when the caller resolved it already. */
+    actor?: Actor;
 }
 
 /**
@@ -64,6 +75,12 @@ export interface WorkerInvokerSeam {
     available?(): boolean;
 }
 
+/**
+ * Why no token could be minted: nobody to run as, no consent to, or the mint
+ * itself failed.
+ */
+export type MintRefusal = 'no-identity' | 'no-consent' | 'mint-failed';
+
 /**
  * Mints the subscriber's app-under-user token for one invocation. The
  * `events:background` consent is what authorizes it, not a narrower
@@ -71,15 +88,35 @@ export interface WorkerInvokerSeam {
  */
 export type SubscriberTokenMinter = (
     invocation: WorkerInvocation,
-) => Promise<string | null>;
+) => Promise<{ token: string } | { refused: MintRefusal }>;
+
+/** Where the handler set is read from when its cache misses. */
+export interface AddressOptions {
+    missRead?: 'primary' | 'replica';
+    /** A handler the set must have; a set without it is re-read on the primary. */
+    handlerName?: string;
+    /** Absent means the app's own set. */
+    handlerScope?: HandlerScopeKind;
+    /** The user whose scope it is, for `handlerScope: 'user'`. */
+    holderUserId?: number;
+}
 
 /**
- * The script an app's handlers deploy as and its invocation key; null when the
- * app has no handlers or no events secret is configured.
+ * The script a handler deploys as and its invocation key; null when it has none
+ * to run, no events secret is configured, or (in a user scope) the holder is
+ * gone or suspended. `holder` is the holder's uuid, for a user scope.
  */
 export type EventsWorkerAddresser = (
     appUid: string,
-) => Promise<{ script: string; key: string } | null>;
+    options?: AddressOptions,
+) => Promise<{ script: string; key: string; holder?: string } | null>;
+
+/** What one run came to, including why nothing was invoked. */
+export type WorkerRunResult =
+    WorkerInvocationResult | { outcome: MintRefusal | 'deploy-busy' };
+
+/** How {@link EventsWorkerInvoker.run} reaches the worker. */
+export type WorkerRunOptions = WorkerInvokeOptions & AddressOptions;
 
 /** How often the same script's retriable failures are logged. */
 const WARN_INTERVAL_MS = 60_000;
@@ -97,6 +134,7 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
     readonly #mintToken: SubscriberTokenMinter;
     readonly #address: EventsWorkerAddresser;
     readonly #available: () => boolean;
+    readonly #userHandlersEnabled: () => boolean;
     /**
      * Script -> when it was last logged, so a failing script floods once a
      * minute.
@@ -108,11 +146,13 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
         mintToken: SubscriberTokenMinter,
         address: EventsWorkerAddresser,
         available: () => boolean,
+        userHandlersEnabled: () => boolean = () => true,
     ) {
         this.#client = client;
         this.#mintToken = mintToken;
         this.#address = address;
         this.#available = available;
+        this.#userHandlersEnabled = userHandlersEnabled;
     }
 
     available(): boolean {
@@ -122,28 +162,66 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
     async invoke(
         invocation: WorkerInvocation,
     ): Promise<WorkerInvocationResult> {
+        const result = await this.run(invocation);
+        switch (result.outcome) {
+            // Nothing was attempted.
+            case 'no-identity':
+            case 'no-consent':
+            case 'mint-failed':
+                return { outcome: 'deferred' };
+            case 'deploy-busy':
+                return { outcome: 'unavailable' };
+            default:
+                return result;
+        }
+    }
+
+    /** {@link invoke}, saying why nothing was invoked. */
+    async run(
+        invocation: WorkerInvocation,
+        options: WorkerRunOptions = {},
+    ): Promise<WorkerRunResult> {
         const { appUid, handlerName } = invocation;
         if (!appUid || !handlerName) return { outcome: 'deferred' };
         if (!this.#available()) return { outcome: 'unavailable' };
+        // Owed, not failed: nothing resolves a user scope while it is off.
+        const { handlerScope } = invocation;
+        if (handlerScope === 'user' && !this.#userHandlersEnabled())
+            return { outcome: 'unavailable' };
 
-        const token = await this.#mintToken(invocation);
-        if (token === null) return { outcome: 'deferred' };
+        const minted = await this.#mintToken(invocation);
+        if ('refused' in minted) return { outcome: minted.refused };
 
         // Nowhere to send it: the app has no handlers or no owner to run them
         // as. Retrying cannot fix that, so the consecutive-failure rule is what
         // stops it.
-        const address = await this.#address(appUid);
+        const address = await this.#address(appUid, {
+            missRead: options.missRead,
+            handlerName,
+            ...(handlerScope === 'user'
+                ? { handlerScope, holderUserId: invocation.holderUserId }
+                : {}),
+        });
         if (address === null) return { outcome: 'retriable' };
 
-        const result = await this.#client.invoke({
-            script: address.script,
-            key: address.key,
-            appUid,
-            handler: handlerName,
-            token,
-            event: invocation.event,
-            ctx: parseContext(invocation.context),
-        });
+        const result = await this.#client.invoke(
+            {
+                script: address.script,
+                key: address.key,
+                appUid,
+                ...(address.holder ? { holder: address.holder } : {}),
+                handler: handlerName,
+                token: minted.token,
+                event: invocation.event,
+                ctx: parseContext(invocation.context),
+            },
+            {
+                lane: options.lane,
+                stampEvent: options.stampEvent,
+                beforeSend: options.beforeSend,
+            },
+        );
+        if (result.deployBusy) return { outcome: 'deploy-busy' };
         if (
             (result.outcome === 'retriable' ||
                 result.outcome === 'unavailable') &&

src/backend/services/events/workers.integration.test.ts

@@ -175,7 +175,7 @@ let destroys: Array<{ appUid: string; ownerId: number | undefined }>;
 
 beforeAll(async () => {
     env = await setupPuterTestEnv({
-        events: { enabled: true },
+        events: { enabled: true, userHandlers: true },
         unlimitedMetering: true,
     } as IConfig);
 
@@ -204,6 +204,7 @@ afterAll(async () => {
 
 beforeEach(async () => {
     await env.server.clients.db.write('DELETE FROM `event_handlers`', []);
+    await env.server.clients.db.write('DELETE FROM `event_user_handlers`', []);
     await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []);
     await env.server.clients.db.write('DELETE FROM `apps`', []);
     creates.length = 0;
@@ -539,6 +540,8 @@ describe('POST /events/workers/destroy', () => {
 
         const suspended = await env.server.stores.durableSubscription.listByHandler(
             app.uid,
+            'app',
+            0,
             'ingestUpload',
             { suspendedReason: 'handler_not_found' },
         );
@@ -554,6 +557,8 @@ describe('POST /events/workers/destroy', () => {
         const stillSuspended =
             await env.server.stores.durableSubscription.listByHandler(
                 app.uid,
+                'app',
+                0,
                 'ingestUpload',
                 { suspendedReason: 'handler_not_found' },
             );
@@ -621,3 +626,319 @@ describe('POST /events/workers/destroy', () => {
         expect(destroys).toEqual([]);
     });
 });
+
+// -- User scopes -----------------------------------------------------
+
+/** `userId`'s own session of `app`, wherever the app's owner is. */
+const sessionIn = async (app: { uid: string }, asUser: 'user' | 'other') => {
+    const { actor } = await env.server.services.auth.authenticate(
+        asUser === 'user' ? env.users.user.token : env.users.other.token,
+    );
+    return env.server.services.auth.getUserAppToken(actor!, app.uid);
+};
+
+const publishUserScope = (token: string, name: string, source: string) =>
+    call('POST', '/events/handlers/publish', token, {
+        scope: 'user',
+        name,
+        source,
+    });
+
+describe('GET /events/workers with user scopes', () => {
+    it('adds the caller`s user scopes to their app workers outside an app', async () => {
+        const mine = await makeApp(userId);
+        await publish(mine.token, {
+            appUid: mine.uid,
+            name: 'a',
+            source: SOURCE,
+        });
+        const theirs = await makeApp(otherUserId);
+        const inTheirs = await sessionIn(theirs, 'user');
+        await publishUserScope(inTheirs, 'x', SOURCE);
+        await publishUserScope(inTheirs, 'y', OTHER_SOURCE);
+        // Another holder's scope in the same app is not the caller's.
+        await publishUserScope(await sessionIn(theirs, 'other'), 'z', SOURCE);
+
+        const listed = await call(
+            'GET',
+            '/events/workers',
+            env.users.user.token,
+        );
+
+        expect(listed.status).toBe(200);
+        expect(listed.body.items).toEqual([
+            expect.objectContaining({
+                scope: 'app',
+                appUid: mine.uid,
+                handlerCount: 1,
+            }),
+            expect.objectContaining({
+                scope: 'user',
+                appUid: theirs.uid,
+                appName: theirs.uid,
+                handlerCount: 2,
+            }),
+        ]);
+        expect(listed.body.cursor).toBeUndefined();
+    });
+
+    it('pages through the app workers and then the user scopes', async () => {
+        const mine = await makeApp(userId);
+        await publish(mine.token, {
+            appUid: mine.uid,
+            name: 'a',
+            source: SOURCE,
+        });
+        const first = await makeApp(otherUserId);
+        const second = await makeApp(otherUserId);
+        await publishUserScope(await sessionIn(first, 'user'), 'x', SOURCE);
+        await publishUserScope(await sessionIn(second, 'user'), 'x', SOURCE);
+
+        const seen: Array<Record<string, unknown>> = [];
+        let cursor: string | undefined;
+        let pages = 0;
+        do {
+            const page = await call(
+                'GET',
+                `/events/workers?limit=1${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`,
+                env.users.user.token,
+            );
+            expect(page.status).toBe(200);
+            seen.push(...(page.body.items as Array<Record<string, unknown>>));
+            cursor = page.body.cursor as string | undefined;
+            pages += 1;
+        } while (cursor && pages < 10);
+
+        expect(seen.map((item) => [item.scope, item.appUid])).toEqual([
+            ['app', mine.uid],
+            ...[first.uid, second.uid].sort().map((uid) => ['user', uid]),
+        ]);
+    });
+
+    it('lists only that app`s scope for the caller inside an app', async () => {
+        const theirs = await makeApp(otherUserId);
+        const otherApp = await makeApp(otherUserId);
+        const inTheirs = await sessionIn(theirs, 'user');
+        await publishUserScope(inTheirs, 'x', SOURCE);
+        await publishUserScope(await sessionIn(otherApp, 'user'), 'x', SOURCE);
+        await publishUserScope(await sessionIn(theirs, 'other'), 'x', SOURCE);
+        // The owner's own worker in that app is not the caller's to see.
+        await publish(theirs.token, {
+            appUid: theirs.uid,
+            name: 'owner',
+            source: SOURCE,
+        });
+
+        const listed = await call('GET', '/events/workers', inTheirs);
+
+        expect(listed.status).toBe(200);
+        expect(listed.body.items).toEqual([
+            expect.objectContaining({
+                scope: 'user',
+                appUid: theirs.uid,
+                handlerCount: 1,
+            }),
+        ]);
+    });
+
+    it('still lists the owner`s worker beside their own user scope inside the app', async () => {
+        const mine = await makeApp(userId);
+        await publish(mine.token, {
+            appUid: mine.uid,
+            name: 'a',
+            source: SOURCE,
+        });
+        await publishUserScope(mine.token, 'x', SOURCE);
+
+        const listed = await call('GET', '/events/workers', mine.token);
+
+        expect(
+            (listed.body.items as Array<Record<string, unknown>>).map(
+                (item) => item.scope,
+            ),
+        ).toEqual(['app', 'user']);
+    });
+
+    it('refuses a scoped token, which has no scope of its own', async () => {
+        const theirs = await makeApp(otherUserId);
+        await publishUserScope(await sessionIn(theirs, 'user'), 'x', SOURCE);
+        const scoped = await appIssuedScopedToken({
+            uid: theirs.uid,
+            token: await sessionIn(theirs, 'user'),
+        });
+
+        const listed = await call('GET', '/events/workers', scoped);
+        expect(listed.status).toBe(403);
+        expect(listed.body.code).toBe('events_worker_owner_only');
+    });
+});
+
+describe('POST /events/workers/destroy with a user scope', () => {
+    it('removes the caller`s scope only, suspends what was bound, and announces it', async () => {
+        const theirs = await makeApp(otherUserId);
+        const mineIn = await sessionIn(theirs, 'user');
+        const otherIn = await sessionIn(theirs, 'other');
+        await publish(theirs.token, {
+            appUid: theirs.uid,
+            name: 'owner',
+            source: SOURCE,
+        });
+        await publishUserScope(mineIn, 'x', SOURCE);
+        await publishUserScope(mineIn, 'y', OTHER_SOURCE);
+        await publishUserScope(otherIn, 'x', SOURCE);
+        destroys.length = 0;
+
+        const destroyed = await call(
+            'POST',
+            '/events/workers/destroy',
+            mineIn,
+            {
+                scope: 'user',
+            },
+        );
+
+        expect(destroyed.status).toBe(200);
+        expect(destroyed.body).toMatchObject({
+            appUid: theirs.uid,
+            removed: 2,
+            suspended: 0,
+            scope: 'user',
+        });
+        const store = env.server.stores.eventHandler;
+        expect(
+            await store.countForApp({ appUid: theirs.uid, holder: userId }),
+        ).toBe(0);
+        expect(
+            await store.countForApp({
+                appUid: theirs.uid,
+                holder: otherUserId,
+            }),
+        ).toBe(1);
+        expect(await store.countForApp({ appUid: theirs.uid, holder: 0 })).toBe(
+            1,
+        );
+        expect(destroys).toEqual([{ appUid: theirs.uid, ownerId: userId }]);
+
+        const again = await call('POST', '/events/workers/destroy', mineIn, {
+            scope: 'user',
+        });
+        expect(again.status).toBe(404);
+    });
+
+    it('refuses a scoped token and an app session naming another app', async () => {
+        const theirs = await makeApp(otherUserId);
+        const mineIn = await sessionIn(theirs, 'user');
+        await publishUserScope(mineIn, 'x', SOURCE);
+        const scoped = await appIssuedScopedToken({
+            uid: theirs.uid,
+            token: mineIn,
+        });
+        const elsewhere = await makeApp(otherUserId);
+
+        for (const [token, body] of [
+            [scoped, { scope: 'user' }],
+            [mineIn, { scope: 'user', appUid: elsewhere.uid }],
+        ] as const) {
+            const refused = await call(
+                'POST',
+                '/events/workers/destroy',
+                token,
+                body,
+            );
+            expect(refused.status).toBe(403);
+            expect(refused.body.code).toBe('events_handler_forbidden');
+        }
+        expect(
+            await env.server.stores.eventHandler.countForApp({
+                appUid: theirs.uid,
+                holder: userId,
+            }),
+        ).toBe(1);
+    });
+});
+
+describe('destroying a user scope from an account session', () => {
+    it('removes the caller`s own scope of any app, and nobody else`s', async () => {
+        const theirs = await makeApp(otherUserId);
+        await publishUserScope(await sessionIn(theirs, 'user'), 'x', SOURCE);
+        await publishUserScope(await sessionIn(theirs, 'other'), 'x', SOURCE);
+        const store = env.server.stores.eventHandler;
+        destroys.length = 0;
+
+        const destroyed = await call(
+            'POST',
+            '/events/workers/destroy',
+            env.users.user.token,
+            { scope: 'user', appUid: theirs.uid },
+        );
+
+        expect(destroyed.status).toBe(200);
+        expect(destroyed.body).toMatchObject({ removed: 1, scope: 'user' });
+        expect(
+            await store.countForApp({ appUid: theirs.uid, holder: userId }),
+        ).toBe(0);
+        expect(
+            await store.countForApp({ appUid: theirs.uid, holder: otherUserId }),
+        ).toBe(1);
+        expect(destroys).toEqual([{ appUid: theirs.uid, ownerId: userId }]);
+
+        // The same call as the other user removes only theirs; the first user
+        // has no scope left to touch.
+        const none = await call(
+            'POST',
+            '/events/workers/destroy',
+            env.users.user.token,
+            { scope: 'user', appUid: theirs.uid },
+        );
+        expect(none.status).toBe(404);
+        expect(
+            await store.countForApp({ appUid: theirs.uid, holder: otherUserId }),
+        ).toBe(1);
+    });
+
+    it('needs an appUid, and still refuses a scoped token', async () => {
+        const theirs = await makeApp(otherUserId);
+        const mineIn = await sessionIn(theirs, 'user');
+        await publishUserScope(mineIn, 'x', SOURCE);
+
+        const noApp = await call(
+            'POST',
+            '/events/workers/destroy',
+            env.users.user.token,
+            { scope: 'user' },
+        );
+        expect(noApp.status).toBe(403);
+
+        const scoped = await appIssuedScopedToken({
+            uid: theirs.uid,
+            token: mineIn,
+        });
+        const refused = await call('POST', '/events/workers/destroy', scoped, {
+            scope: 'user',
+            appUid: theirs.uid,
+        });
+        expect(refused.status).toBe(403);
+    });
+
+    it('works with user handlers switched off, so a user can stop paying', async () => {
+        const theirs = await makeApp(otherUserId);
+        await publishUserScope(await sessionIn(theirs, 'user'), 'x', SOURCE);
+        const config = (
+            env.server.services.events as unknown as {
+                config: { events: { userHandlers?: boolean } };
+            }
+        ).config;
+        config.events.userHandlers = false;
+        try {
+            const destroyed = await call(
+                'POST',
+                '/events/workers/destroy',
+                env.users.user.token,
+                { scope: 'user', appUid: theirs.uid },
+            );
+            expect(destroyed.status).toBe(200);
+        } finally {
+            config.events.userHandlers = true;
+        }
+    });
+});

src/backend/services/index.ts

@@ -36,6 +36,7 @@ import { FSService } from './fs/FSService';
 import { KvShareService } from './kvShare/KvShareService';
 import { ServerHealthService } from './health/ServerHealthService';
 import { PuterHomepageService } from './homepage/PuterHomepageService';
+import { JobScheduler } from './jobs/JobScheduler';
 import { LocalWorkerService } from './localworker/LocalWorkerService';
 import { MeteringService } from './metering/MeteringService';
 import { NotificationService } from './notification/NotificationService';
@@ -60,6 +61,7 @@ import { UserAccountService } from './user/UserAccountService';
  */
 declare module './types' {
     interface IPuterServiceInstances {
+        jobs: JobScheduler;
         metering: MeteringService;
         appOriginBlocklist: AppOriginBlocklistService;
         permission: PermissionService;
@@ -103,6 +105,9 @@ declare module './types' {
 // NotificationService depends on notification store (for DB) + event client (for socket push).
 // BroadcastService is independent — only needs the event client.
 export const puterServices = {
+    // First, so every later service can register jobs with it. Reaches
+    // `broadcast` at call time only.
+    jobs: JobScheduler,
     metering: MeteringService,
     // Declared before `auth` so AuthService sees it as a prior peer — it
     // queries the blocklist on app-token acquisition and per-request app

src/backend/services/jobs/JobScheduler.test.ts

@@ -0,0 +1,781 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { randomUUID } from 'node:crypto';
+import MockRedis from 'ioredis-mock';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import {
+    JobLeaseLostError,
+    JobScheduler,
+    type JobRunContext,
+} from './JobScheduler.js';
+
+type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
+
+const HOUR = 60 * 60_000;
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+describe('JobScheduler', () => {
+    let server: PuterServer;
+    const schedulers: JobScheduler[] = [];
+
+    beforeAll(async () => {
+        server = await setupTestServer();
+    });
+
+    afterAll(async () => {
+        await server?.shutdown();
+    });
+
+    afterEach(() => {
+        vi.restoreAllMocks();
+        vi.useRealTimers();
+        for (const scheduler of schedulers.splice(0)) {
+            scheduler.onServerPrepareShutdown();
+        }
+    });
+
+    const db = () => server.clients.db;
+
+    /**
+     * A node of its own: its own config and region, on the test server's Redis
+     * unless given one.
+     */
+    const node = ({
+        namespace,
+        region = 'r1',
+        redis,
+        domain = 'puter.test',
+        enabled,
+    }: {
+        namespace?: string;
+        region?: string;
+        redis?: InstanceType<typeof MockRedis>;
+        domain?: string;
+        enabled?: boolean;
+    } = {}) => {
+        const config = {
+            port: 0,
+            extensions: [],
+            domain,
+            serverId: `node-${schedulers.length}`,
+            jobs: { namespace, enabled },
+        } as IConfig;
+        const clients = (
+            redis ? { ...server.clients, redis } : server.clients
+        ) as SchedulerArgs[1];
+        const services = {
+            broadcast: { regionId: region },
+        } as unknown as SchedulerArgs[3];
+        const scheduler = new JobScheduler(
+            config,
+            clients,
+            server.stores as SchedulerArgs[2],
+            services,
+        );
+        schedulers.push(scheduler);
+        return scheduler;
+    };
+
+    const freshNamespace = () => `ns-${randomUUID()}`;
+
+    /**
+     * A job that runs `batches` batches `batchMs` apart, checking its signal
+     * before each.
+     */
+    const batchedJob = (batches: number, batchMs: number) => {
+        const seen = { done: 0, signal: null as AbortSignal | null };
+        const run = async ({ signal }: JobRunContext) => {
+            seen.signal = signal;
+            for (let i = 0; i < batches; i++) {
+                if (signal.aborted) return;
+                await sleep(batchMs);
+                seen.done += 1;
+            }
+        };
+        return { seen, run };
+    };
+
+    const keysOf = (namespace: string, name: string) => ({
+        lock: `jobs:{${namespace}:${name}}:lock`,
+        gate: `jobs:{${namespace}:${name}}:next`,
+        run: `jobs:{${namespace}:${name}}:run`,
+    });
+
+    // -- Registration -------------------------------------------------
+
+    it('refuses a bad or duplicate registration', () => {
+        const scheduler = node({ namespace: freshNamespace() });
+        const run = async () => {};
+        expect(() =>
+            scheduler.register('bad name', {
+                every: 1000,
+                scope: 'fleet',
+                run,
+            }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('job', { every: 0, scope: 'fleet', run }),
+        ).toThrow(/every/);
+        expect(() =>
+            scheduler.register('job', {
+                every: 1000,
+                scope: 'fleet',
+                leaseMs: 1.5,
+                run,
+            }),
+        ).toThrow(/leaseMs/);
+        // `:` would let two (namespace, name) pairs share Redis keys.
+        expect(() =>
+            scheduler.register('b:c', { every: 1000, scope: 'fleet', run }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('x'.repeat(129), {
+                every: 1000,
+                scope: 'fleet',
+                run,
+            }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('job.region', {
+                every: 1000,
+                scope: 'region',
+                leaseMs: 2000,
+                run,
+            }),
+        ).toThrow(/exceeds every/);
+        scheduler.register('job', { every: 1000, scope: 'fleet', run });
+        expect(() =>
+            scheduler.register('job', { every: 1000, scope: 'fleet', run }),
+        ).toThrow(/already registered/);
+    });
+
+    it('falls back to the default namespace, once and loudly, when the configured one cannot be held', () => {
+        const run = async () => {};
+        const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+        for (const namespace of ['a:b', 'x'.repeat(129), 'spaced name']) {
+            const scheduler = node({ namespace });
+            scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
+            expect(scheduler.namespace).toBe('default');
+        }
+        logged.mockClear();
+
+        // A self-host whose domain carries a port still boots.
+        const scheduler = node({ domain: 'localhost:4100' });
+        scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
+        scheduler.register('job.b', { every: 1000, scope: 'fleet', run });
+        expect(scheduler.namespace).toBe('default');
+        expect(logged).toHaveBeenCalledTimes(1);
+        expect(String(logged.mock.calls[0][0])).toMatch(
+            /localhost:4100.*jobs\.namespace/,
+        );
+    });
+
+    it('does nothing at all while no job is registered', async () => {
+        vi.useFakeTimers({
+            toFake: [
+                'setInterval',
+                'setTimeout',
+                'clearInterval',
+                'clearTimeout',
+            ],
+        });
+        // Timers the test runtime itself already holds.
+        const baseline = vi.getTimerCount();
+        const reads = vi.spyOn(db(), 'read');
+        const preads = vi.spyOn(db(), 'pread');
+        const writes = vi.spyOn(db(), 'write');
+        const scheduler = node({ namespace: freshNamespace() });
+
+        scheduler.onServerStart();
+        expect(vi.getTimerCount()).toBe(baseline);
+        await scheduler.poll();
+        expect(reads).not.toHaveBeenCalled();
+        expect(preads).not.toHaveBeenCalled();
+        expect(writes).not.toHaveBeenCalled();
+
+        // A job registered after start arms the poll; shutdown disarms it.
+        scheduler.register('late', {
+            every: HOUR,
+            scope: 'fleet',
+            run: async () => {},
+        });
+        expect(vi.getTimerCount()).toBeGreaterThan(baseline);
+        scheduler.onServerPrepareShutdown();
+        expect(vi.getTimerCount()).toBe(baseline);
+    });
+
+    it('claims nothing once draining, and leaves a running job its lease', async () => {
+        const scheduler = node({ namespace: `drain${Date.now()}` });
+        let release!: () => void;
+        let signal!: AbortSignal;
+        scheduler.register('long', {
+            every: HOUR,
+            scope: 'region',
+            run: (ctx) => {
+                signal = ctx.signal;
+                return new Promise<void>((resolve) => {
+                    release = resolve;
+                });
+            },
+        });
+        const first = scheduler.poll();
+        await vi.waitFor(() => expect(release).toBeDefined());
+        const run = vi.fn(async () => {});
+        scheduler.register('other', { every: HOUR, scope: 'region', run });
+
+        scheduler.onServerDrainStart();
+        await scheduler.poll();
+        expect(run).not.toHaveBeenCalled();
+        expect(signal.aborted).toBe(false);
+
+        release();
+        await first;
+    });
+
+    it('arms no poll on a node with jobs turned off, though poll() still runs them', async () => {
+        vi.useFakeTimers({
+            toFake: [
+                'setInterval',
+                'setTimeout',
+                'clearInterval',
+                'clearTimeout',
+            ],
+        });
+        const baseline = vi.getTimerCount();
+        const scheduler = node({ namespace: freshNamespace(), enabled: false });
+        const warned = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let runs = 0;
+        scheduler.onServerStart();
+        scheduler.register('job.off', {
+            every: HOUR,
+            scope: 'fleet',
+            run: async () => {
+                runs += 1;
+            },
+        });
+        scheduler.register('job.off.too', {
+            every: HOUR,
+            scope: 'region',
+            run: async () => undefined,
+        });
+        expect(vi.getTimerCount()).toBe(baseline);
+        // Said once per node, not per job.
+        expect(
+            warned.mock.calls.filter(([line]) =>
+                String(line).includes('jobs.enabled'),
+            ),
+        ).toHaveLength(1);
+        vi.useRealTimers();
+
+        await scheduler.poll();
+        expect(runs).toBe(1);
+    });
+
+    describe('on a controlled clock', () => {
+        const leaseMs = 300;
+        /**
+         * Pending timeouts armed from JobScheduler.ts. The shared test server's
+         * own loops keep running on real intervals and may arm fake timers
+         * mid-test, so the global timer count says nothing about the
+         * scheduler.
+         */
+        let armed: Set<unknown>;
+        const schedulerTimers = () => armed.size;
+
+        beforeEach(() => {
+            vi.useFakeTimers({
+                toFake: [
+                    'setTimeout',
+                    'clearTimeout',
+                    'setInterval',
+                    'clearInterval',
+                    'performance',
+                ],
+            });
+            armed = new Set();
+            const fakeSetTimeout = globalThis.setTimeout;
+            const fakeClearTimeout = globalThis.clearTimeout;
+            const fromScheduler = () => {
+                const limit = Error.stackTraceLimit;
+                Error.stackTraceLimit = 30;
+                const stack = new Error().stack ?? '';
+                Error.stackTraceLimit = limit;
+                return /services[\\/]jobs[\\/]JobScheduler\.ts/.test(stack);
+            };
+            vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+                fn: (...args: unknown[]) => void,
+                ms?: number,
+                ...rest: unknown[]
+            ) => {
+                if (!fromScheduler()) return fakeSetTimeout(fn, ms, ...rest);
+                const handle = fakeSetTimeout(
+                    (...args: unknown[]) => {
+                        armed.delete(handle);
+                        fn(...args);
+                    },
+                    ms,
+                    ...rest,
+                );
+                armed.add(handle);
+                return handle;
+            }) as never);
+            vi.spyOn(globalThis, 'clearTimeout').mockImplementation(((
+                handle: Parameters<typeof clearTimeout>[0],
+            ) => {
+                armed.delete(handle);
+                fakeClearTimeout(handle);
+            }) as never);
+        });
+
+        /** Batches of `batchMs`, each started only while the signal is live. */
+        const timedJob = (batches: number, batchMs: number) => {
+            const seen = {
+                starts: [] as number[],
+                abortedAt: null as number | null,
+                signal: null as AbortSignal | null,
+                saveError: null as unknown,
+            };
+            const run = async ({ signal, saveState }: JobRunContext) => {
+                seen.signal = signal;
+                signal.addEventListener('abort', () => {
+                    seen.abortedAt = performance.now();
+                });
+                for (let i = 0; i < batches; i++) {
+                    if (signal.aborted) break;
+                    seen.starts.push(performance.now());
+                    await sleep(batchMs);
+                }
+                try {
+                    await saveState({ late: true });
+                } catch (err) {
+                    seen.saveError = err;
+                }
+            };
+            return { seen, run };
+        };
+
+        /** Let the run play out, then check nothing it armed is left behind. */
+        const settle = async (running: Promise<void>) => {
+            await vi.advanceTimersByTimeAsync(2_000);
+            await running;
+            expect(schedulerTimers()).toBe(0);
+        };
+
+        it('clears a run’s timers at shutdown even while the run hangs', async () => {
+            const scheduler = node({ namespace: freshNamespace() });
+            let release!: () => void;
+            scheduler.register('job.hang', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run: () =>
+                    new Promise<void>((resolve) => {
+                        release = resolve;
+                    }),
+            });
+            const running = scheduler.poll();
+            await vi.advanceTimersByTimeAsync(10);
+            expect(schedulerTimers()).toBeGreaterThan(0);
+
+            scheduler.onServerPrepareShutdown();
+            expect(schedulerTimers()).toBe(0);
+            release();
+            await settle(running);
+        });
+
+        it('never starts a region run once taking the lock used up half the lease', async () => {
+            const namespace = freshNamespace();
+            const redis = new MockRedis({ port: 17104 });
+            const scheduler = node({ namespace, redis });
+            const run = vi.fn(async () => {});
+            scheduler.register('job.slowLock', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run,
+            });
+            const set = redis.set.bind(redis) as (
+                ...args: unknown[]
+            ) => Promise<unknown>;
+            const slowLock = vi.spyOn(redis, 'set').mockImplementation((async (
+                ...args: unknown[]
+            ) => {
+                const result = await set(...args);
+                if (String(args[0]).endsWith(':lock')) {
+                    await sleep(leaseMs / 2 + 10);
+                }
+                return result;
+            }) as never);
+            await settle(scheduler.poll());
+
+            expect(run).not.toHaveBeenCalled();
+            // The gate went back, so the next poll runs it.
+            expect(
+                await redis.exists(`jobs:{${namespace}:job.slowLock}:next`),
+            ).toBe(0);
+            slowLock.mockRestore();
+            await settle(scheduler.poll());
+            expect(run).toHaveBeenCalledTimes(1);
+        });
+
+        it('aborts a region run before its next batch once the lock lapses, and leaves it due at once', async () => {
+            const namespace = freshNamespace();
+            const redis = new MockRedis({ port: 17106 });
+            const scheduler = node({ namespace, redis });
+            const job = timedJob(10, 70);
+            scheduler.register('job.lapse', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run: job.run,
+            });
+            const lockedAt = performance.now();
+            await settle(scheduler.poll());
+
+            expect(job.seen.abortedAt! - lockedAt).toBe(leaseMs);
+            expect((job.seen.signal?.reason as Error).message).toMatch(
+                /lapsed/,
+            );
+            expect(job.seen.starts.length).toBeLessThan(10);
+            expect(job.seen.starts.every((t) => t < job.seen.abortedAt!)).toBe(
+                true,
+            );
+            // The gate is still this run's, so its last save went through;
+            // interrupted, the gate goes back and the job is due at once.
+            expect(job.seen.saveError).toBeNull();
+            const keys = {
+                gate: `jobs:{${namespace}:job.lapse}:next`,
+                run: `jobs:{${namespace}:job.lapse}:run`,
+            };
+            expect(await redis.exists(keys.gate)).toBe(0);
+            expect(await redis.hget(keys.run, 'state')).toBe('{"late":true}');
+            expect(await redis.hget(keys.run, 'failures')).toBeNull();
+        });
+    });
+
+    it('refuses state over 64 KB', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17107 });
+        const scheduler = node({ namespace, redis });
+        let refused: unknown = null;
+        scheduler.register('job.bigState', {
+            every: HOUR,
+            scope: 'region',
+            run: async ({ saveState }) => {
+                try {
+                    await saveState({ blob: 'x'.repeat(70_000) });
+                } catch (err) {
+                    refused = err;
+                }
+            },
+        });
+        await scheduler.poll();
+        expect((refused as Error).message).toMatch(/over 65536 bytes/);
+        expect(
+            await redis.hget(keysOf(namespace, 'job.bigState').run, 'state'),
+        ).toBeNull();
+    });
+
+    it('releases an interrupted run on shutdown without counting a failure', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17109 });
+        const scheduler = node({ namespace, redis });
+        const job = batchedJob(50, 50);
+        scheduler.register('job.shutdown', {
+            every: HOUR,
+            scope: 'region',
+            run: job.run,
+        });
+        const running = scheduler.poll();
+        await vi.waitFor(() => expect(job.seen.done).toBeGreaterThan(0));
+        scheduler.onServerPrepareShutdown();
+        await running;
+
+        expect(job.seen.done).toBeLessThan(50);
+        expect((job.seen.signal?.reason as Error).message).toBe(
+            'server shutting down',
+        );
+        const keys = keysOf(namespace, 'job.shutdown');
+        // Due again at once, with no failure counted.
+        expect(await redis.exists(keys.gate)).toBe(0);
+        expect(await redis.hget(keys.run, 'failures')).toBeNull();
+    });
+
+    it('keeps namespaces apart: the same job name runs once in each', async () => {
+        const redis = new MockRedis({ port: 17110 });
+        const [nsA, nsB] = [freshNamespace(), freshNamespace()];
+        const nodes = [
+            node({ namespace: nsA, redis }),
+            node({ namespace: nsA, redis }),
+            node({ namespace: nsB, redis }),
+            node({ namespace: nsB, redis }),
+        ];
+        const runs: string[] = [];
+        nodes.forEach((scheduler, i) =>
+            scheduler.register('job.shared', {
+                every: HOUR,
+                scope: 'region',
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    await sleep(20);
+                },
+            }),
+        );
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+    });
+
+    it('falls back to the domain as its namespace', () => {
+        expect(node({ domain: 'example.test' }).namespace).toBe('example.test');
+        expect(
+            node({ namespace: 'explicit', domain: 'example.test' }).namespace,
+        ).toBe('explicit');
+    });
+
+    it('boots a server whose domain carries a port, registering its jobs under the default namespace', async () => {
+        const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+        const portDomain = await setupTestServer({
+            domain: 'localhost:4101',
+        } as IConfig);
+        try {
+            expect(portDomain.services.jobs.namespace).toBe('default');
+            const namespaceErrors = logged.mock.calls.filter(([message]) =>
+                String(message).startsWith('[jobs] namespace'),
+            );
+            expect(namespaceErrors).toHaveLength(1);
+        } finally {
+            await portDomain.shutdown();
+        }
+    });
+
+    // -- Region scope -------------------------------------------------
+
+    it('runs a region job once per region', async () => {
+        const namespace = freshNamespace();
+        const redisA = new MockRedis({ port: 17101 });
+        const redisB = new MockRedis({ port: 17102 });
+        const nodes = [
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'b', redis: redisB }),
+            node({ namespace, region: 'b', redis: redisB }),
+        ];
+        const runs: string[] = [];
+        nodes.forEach((scheduler, i) =>
+            scheduler.register('job.region', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs: 5_000,
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    await sleep(20);
+                },
+            }),
+        );
+
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+        expect(
+            await redisA.pttl(`jobs:{${namespace}:job.region}:next`),
+        ).toBeGreaterThan(HOUR - 60_000);
+        // The region lock was released after the run.
+        expect(await redisA.exists(`jobs:{${namespace}:job.region}:lock`)).toBe(
+            0,
+        );
+    });
+
+    it('backs off a failing region job and keeps its state in Redis', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17103 });
+        const scheduler = node({ namespace, redis });
+        const seen: unknown[] = [];
+        let fail = true;
+        scheduler.register('job.regionFail', {
+            every: HOUR,
+            scope: 'region',
+            leaseMs: 5_000,
+            run: async ({ state, saveState }) => {
+                seen.push(state);
+                await saveState({ page: seen.length });
+                if (fail) throw new Error('nope');
+            },
+        });
+        const gate = `jobs:{${namespace}:job.regionFail}:next`;
+
+        await scheduler.poll();
+        expect(
+            await redis.hget(
+                `jobs:{${namespace}:job.regionFail}:run`,
+                'failures',
+            ),
+        ).toBe('1');
+        expect(await redis.pttl(gate)).toBeLessThanOrEqual(2_000);
+
+        fail = false;
+        await redis.del(gate);
+        await scheduler.poll();
+        expect(seen).toEqual([null, { page: 1 }]);
+        expect(
+            await redis.hget(
+                `jobs:{${namespace}:job.regionFail}:run`,
+                'failures',
+            ),
+        ).toBeNull();
+        expect(await redis.pttl(gate)).toBeGreaterThan(HOUR - 60_000);
+    });
+
+    it('skips region jobs quietly while Redis is unreachable', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17108 });
+        const scheduler = node({ namespace, redis });
+        const run = vi.fn(async () => {});
+        scheduler.register('job.noRedis', {
+            every: HOUR,
+            scope: 'region',
+            run,
+        });
+        vi.spyOn(redis, 'exists').mockRejectedValue(
+            new Error('connection refused'),
+        );
+        const errors = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+        await scheduler.poll();
+        expect(run).not.toHaveBeenCalled();
+        expect(errors).not.toHaveBeenCalled();
+    });
+
+    it('fences a region run that outlived its gate off its successor’s keys', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17105 });
+        const [slow, next] = [
+            node({ namespace, redis }),
+            node({ namespace, redis }),
+        ];
+        const keys = {
+            gate: `jobs:{${namespace}:job.fenced}:next`,
+            run: `jobs:{${namespace}:job.fenced}:run`,
+        };
+        let release!: () => void;
+        let slowStarted = false;
+        let slowSaveError: unknown = null;
+        // Ignores its signal and outlives both its 200 ms lock and 400 ms gate.
+        slow.register('job.fenced', {
+            every: 400,
+            scope: 'region',
+            leaseMs: 200,
+            run: async ({ saveState }) => {
+                slowStarted = true;
+                await new Promise<void>((resolve) => {
+                    release = resolve;
+                });
+                try {
+                    await saveState({ by: 'slow' });
+                } catch (err) {
+                    slowSaveError = err;
+                }
+                throw new Error('slow run failed');
+            },
+        });
+        next.register('job.fenced', {
+            every: 400,
+            scope: 'region',
+            leaseMs: 200,
+            run: async ({ saveState }) => {
+                await saveState({ by: 'next' });
+            },
+        });
+
+        const slowRun = slow.poll();
+        await vi.waitFor(() => expect(slowStarted).toBe(true));
+        await sleep(450);
+        await next.poll();
+        const successorGate = await redis.get(keys.gate);
+        expect(successorGate).not.toBeNull();
+
+        release();
+        await slowRun;
+        expect(slowSaveError).toBeInstanceOf(JobLeaseLostError);
+        // Neither the slow run's state, its failure count, nor its cleanup
+        // touched what the successor holds.
+        expect(await redis.get(keys.gate)).toBe(successorGate);
+        expect(await redis.pttl(keys.gate)).toBeGreaterThan(0);
+        expect(await redis.hget(keys.run, 'state')).toBe('{"by":"next"}');
+        expect(await redis.hget(keys.run, 'failures')).toBeNull();
+    });
+    it('runs a fleet job once per region, its lease capped at its cadence and nearPrimary ignored', async () => {
+        const namespace = freshNamespace();
+        const redisA = new MockRedis({ port: 17111 });
+        const redisB = new MockRedis({ port: 17112 });
+        const nodes = [
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'b', redis: redisB }),
+            node({ namespace, region: 'b', redis: redisB }),
+        ];
+        const keys = keysOf(namespace, 'job.fleet');
+        const runs: string[] = [];
+        const lockTtls: number[] = [];
+        const reads = vi.spyOn(db(), 'read');
+        const preads = vi.spyOn(db(), 'pread');
+        const writes = vi.spyOn(db(), 'write');
+        nodes.forEach((scheduler, i) => {
+            const redis = i < 2 ? redisA : redisB;
+            scheduler.register('job.fleet', {
+                every: 2_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                // Longer than `every`: refused for a region job, capped here.
+                leaseMs: HOUR,
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    lockTtls.push(await redis.pttl(keys.lock));
+                    await sleep(20);
+                },
+            });
+        });
+
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+        for (const ttl of lockTtls) {
+            expect(ttl).toBeGreaterThan(0);
+            expect(ttl).toBeLessThanOrEqual(2_000);
+        }
+        expect(await redisA.pttl(keys.gate)).toBeGreaterThan(1_000);
+        expect(await redisB.pttl(keys.gate)).toBeGreaterThan(1_000);
+        expect(reads).not.toHaveBeenCalled();
+        expect(preads).not.toHaveBeenCalled();
+        expect(writes).not.toHaveBeenCalled();
+    });
+});

src/backend/services/jobs/JobScheduler.ts

@@ -0,0 +1,528 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { withRedisLock } from '../../util/redisLock.js';
+import { resolveJobsNamespace } from '../../util/jobsNamespace.js';
+import { deliveryBackoffMs } from '../../util/retryBackoff.js';
+import { PuterService } from '../types';
+import {
+    JobLeaseLostError,
+    asError,
+    lostLease,
+    parseState,
+    runOutcome,
+    serializeState,
+    type RegisteredJob,
+    type RunLease,
+} from './jobRun.js';
+
+export { JobLeaseLostError } from './jobRun.js';
+
+/** How often each node looks for due jobs. */
+const JOB_POLL_MS = 30_000;
+/** Each poll waits up to this long first, so nodes don't all claim at once. */
+const JOB_POLL_JITTER_MS = 5_000;
+const JOB_DEFAULT_LEASE_MS = 15 * 60_000;
+/** Consecutive failures that raise `scheduled_job_failing`. */
+const JOB_FAILING_ALARM_AT = 3;
+const JOB_SHUTDOWN_WAIT_MS = 10_000;
+/**
+ * Job names: ASCII, at most 128, and without `:`, so a (namespace, name) pair
+ * maps to exactly one set of Redis keys (namespaces follow the same rule).
+ */
+const JOB_ID_PATTERN = /^[A-Za-z0-9._-]{1,128}$/;
+
+// Region writes apply only while the gate still holds this run's token.
+const REGION_FENCED_WRITE = `
+if redis.call('get', KEYS[1]) ~= ARGV[1] then return -1 end
+local op = ARGV[2]
+if op == 'hset' then return redis.call('hset', KEYS[2], ARGV[3], ARGV[4]) end
+if op == 'hdel' then return redis.call('hdel', KEYS[2], ARGV[3]) end
+if op == 'hincrby' then return redis.call('hincrby', KEYS[2], ARGV[3], 1) end
+if op == 'pexpire' then return redis.call('pexpire', KEYS[1], ARGV[3]) end
+if op == 'del' then return redis.call('del', KEYS[1]) end
+return -1
+`;
+
+/**
+ * `region` runs once per cadence per region. `fleet` means once across every
+ * region where `services.jobs` supports it; this default runs it per region.
+ */
+export type JobScope = 'fleet' | 'region';
+
+export interface JobRunContext {
+    /** Aborts once the run may have lost the job. Check it before each batch. */
+    signal: AbortSignal;
+    /** What an earlier run saved, or null. */
+    state: unknown;
+    /**
+     * Persist JSON-serializable `state` for later runs. Throws once the run
+     * lost the job.
+     */
+    saveState: (state: unknown) => Promise<void>;
+}
+
+export interface JobDefinition {
+    /** Cadence in ms. */
+    every: number;
+    scope: JobScope;
+    /**
+     * Prefer the region nearest the primary database, for a `fleet` job. The
+     * default scheduler ignores it.
+     */
+    nearPrimary?: boolean;
+    /**
+     * How long a run holds the job, in total on the region path (at most
+     * `every`). Default 15 min, capped at `every` for `region`; a `fleet` job
+     * run per region is capped at `every` too.
+     */
+    leaseMs?: number;
+    run: (ctx: JobRunContext) => Promise<void>;
+}
+
+/** The `afterId` cursor a keyset-walking job saved in its state, or 0. */
+export const savedAfterId = (state: unknown): number => {
+    const afterId = (state as { afterId?: unknown } | null)?.afterId;
+    return typeof afterId === 'number' &&
+        Number.isSafeInteger(afterId) &&
+        afterId > 0
+        ? afterId
+        : 0;
+};
+
+interface RegionKeys {
+    lock: string;
+    gate: string;
+    run: string;
+}
+
+/**
+ * Runs our own recurring backend work once per cadence per region, under a
+ * Redis lock. `fleet` jobs take the same path here; an extension registered as
+ * `services.jobs` can run them fleet-wide by overriding `pollFleet`. Nothing
+ * polls until a job is registered.
+ */
+export class JobScheduler extends PuterService {
+    #jobs = new Map<string, RegisteredJob>();
+    #running = new Map<string, Promise<void>>();
+    #leases = new Set<RunLease>();
+    #started = false;
+    #stopping = false;
+    /** Polling stopped; runs already going keep their leases. */
+    #draining = false;
+    #timer: ReturnType<typeof setInterval> | null = null;
+    #pollDelays = new Set<ReturnType<typeof setTimeout>>();
+    #warnedOff = false;
+    #namespace: string | null = null;
+
+    /** Keys this deployment's jobs, so deployments sharing Redis stay apart. */
+    get namespace(): string {
+        return (this.#namespace ??= resolveJobsNamespace(this.config));
+    }
+
+    register(name: string, definition: JobDefinition): void {
+        if (!JOB_ID_PATTERN.test(name)) {
+            throw new Error(`invalid job name: ${name}`);
+        }
+        if (this.#jobs.has(name)) {
+            throw new Error(`job already registered: ${name}`);
+        }
+        const { every, scope } = definition;
+        if (!Number.isSafeInteger(every) || every <= 0) {
+            throw new Error(`job ${name}: every must be a positive integer`);
+        }
+        if (scope !== 'fleet' && scope !== 'region') {
+            throw new Error(`job ${name}: unknown scope`);
+        }
+        const leaseMs =
+            definition.leaseMs ??
+            (scope === 'region'
+                ? Math.min(JOB_DEFAULT_LEASE_MS, every)
+                : JOB_DEFAULT_LEASE_MS);
+        if (!Number.isSafeInteger(leaseMs) || leaseMs <= 0) {
+            throw new Error(`job ${name}: leaseMs must be a positive integer`);
+        }
+        // The gate that fences a region run lasts `every`.
+        if (scope === 'region' && leaseMs > every) {
+            throw new Error(
+                `job ${name}: a region job's leaseMs exceeds every`,
+            );
+        }
+        this.#jobs.set(name, {
+            every,
+            scope,
+            nearPrimary: definition.nearPrimary ?? false,
+            leaseMs,
+            run: definition.run,
+        });
+        if (this.#started) this.#arm();
+    }
+
+    override onServerStart(): void {
+        this.#started = true;
+        if (this.#jobs.size > 0) this.#arm();
+    }
+
+    /** Claim nothing new while draining; running jobs carry on until prepare. */
+    override onServerDrainStart(): void {
+        this.#stopPolling();
+    }
+
+    override onServerPrepareShutdown(): void {
+        this.#stopping = true;
+        this.#stopPolling();
+        for (const lease of [...this.#leases]) {
+            lease.controller.abort(
+                new JobLeaseLostError('server shutting down'),
+            );
+            lease.close();
+        }
+    }
+
+    #stopPolling(): void {
+        this.#draining = true;
+        if (this.#timer) clearInterval(this.#timer);
+        this.#timer = null;
+        for (const delay of this.#pollDelays) clearTimeout(delay);
+        this.#pollDelays.clear();
+    }
+
+    /**
+     * Give aborted runs a moment to record themselves before the database
+     * closes.
+     */
+    override async onServerShutdown(): Promise<void> {
+        if (this.#running.size === 0) return;
+        let wait: ReturnType<typeof setTimeout> | undefined;
+        await Promise.race([
+            Promise.allSettled(this.#running.values()),
+            new Promise((resolve) => {
+                wait = setTimeout(resolve, JOB_SHUTDOWN_WAIT_MS);
+            }),
+        ]);
+        clearTimeout(wait);
+    }
+
+    /**
+     * Start every registered job that is due and not already running here.
+     * Resolves once the runs it started have finished.
+     */
+    async poll(): Promise<void> {
+        if (this.#stopping || this.#draining) return;
+        const fleet: string[] = [];
+        const region: Promise<void>[] = [];
+        for (const [name, job] of this.#jobs) {
+            if (this.#running.has(name)) continue;
+            if (job.scope === 'fleet') {
+                fleet.push(name);
+                continue;
+            }
+            region.push(this.track(name, () => this.#tryRegion(name, job)));
+        }
+        await Promise.all([this.pollFleet(fleet), ...region]);
+    }
+
+    #arm(): void {
+        if (this.#timer || this.#stopping || this.#draining) return;
+        if (this.config.jobs?.enabled === false) {
+            // Nothing alarms on a job that never runs, so say it here.
+            if (!this.#warnedOff) {
+                console.warn(
+                    '[jobs] polling is off on this node (jobs.enabled: false); other nodes must run the jobs',
+                );
+            }
+            this.#warnedOff = true;
+            return;
+        }
+        const tick = () => {
+            const delay = setTimeout(() => {
+                this.#pollDelays.delete(delay);
+                void this.#pollSafely();
+            }, Math.random() * JOB_POLL_JITTER_MS);
+            delay.unref?.();
+            this.#pollDelays.add(delay);
+        };
+        const timer = setInterval(tick, JOB_POLL_MS);
+        timer.unref?.();
+        this.#timer = timer;
+        tick();
+    }
+
+    async #pollSafely(): Promise<void> {
+        try {
+            await this.poll();
+        } catch (err) {
+            console.error('[jobs] poll failed', err);
+        }
+    }
+
+    // -- Region scope -------------------------------------------------
+
+    async #tryRegion(name: string, job: RegisteredJob): Promise<void> {
+        const redis = this.clients.redis;
+        const keys = regionKeys(this.namespace, name);
+        let gated: number;
+        try {
+            gated = await redis.exists(keys.gate);
+        } catch {
+            // Redis is unreachable; the lock below would be busy anyway.
+            return;
+        }
+        if (gated > 0) return;
+        // Before the lock, so acquiring it counts against `leaseMs`.
+        const startedAt = performance.now();
+        await withRedisLock(
+            redis,
+            keys.lock,
+            async () => {
+                const token = randomUUID();
+                const gate = await redis.set(
+                    keys.gate,
+                    token,
+                    'PX',
+                    job.every,
+                    'NX',
+                );
+                if (gate !== 'OK') return;
+                await this.#runRegion(name, job, keys, token, startedAt);
+            },
+            {
+                ttlMs: job.leaseMs,
+                onUnavailable: 'busy',
+                onBusy: (): void => undefined,
+            },
+        );
+    }
+
+    /**
+     * One run under the region lock. The lock doesn't renew, so the signal
+     * aborts once it lapses, and every write is fenced on the gate's token.
+     */
+    async #runRegion(
+        name: string,
+        job: RegisteredJob,
+        keys: RegionKeys,
+        token: string,
+        startedAt: number,
+    ): Promise<void> {
+        const lease = this.#regionLease(job.leaseMs, startedAt);
+        const { signal } = lease.controller;
+        const write = (op: string, ...args: (string | number)[]) =>
+            this.#regionWrite(keys, token, op, ...args);
+        let error: unknown = null;
+        try {
+            const saved = await this.clients.redis.hget(keys.run, 'state');
+            if (await lease.ready()) {
+                await job.run({
+                    signal,
+                    state: parseState(saved),
+                    saveState: async (state) => {
+                        const json = serializeState(state);
+                        if ((await write('hset', 'state', json)) < 0) {
+                            lostLease(lease, name);
+                        }
+                    },
+                });
+            }
+        } catch (err) {
+            error = err;
+        } finally {
+            lease.close();
+        }
+
+        const outcome = runOutcome(signal, error);
+        if (outcome === 'success') {
+            await write('hdel', 'failures');
+        } else if (outcome === 'failure') {
+            const failures = await write('hincrby', 'failures');
+            if (failures < 0) return;
+            await write(
+                'pexpire',
+                Math.min(job.every, deliveryBackoffMs(failures)),
+            );
+            this.reportFailure(name, failures, error);
+        } else {
+            await write('del');
+        }
+    }
+
+    /**
+     * The lock lapses `leaseMs` after `from`; the run needs half of it left to
+     * start.
+     */
+    #regionLease(leaseMs: number, from: number): RunLease {
+        const controller = new AbortController();
+        const deadline = from + leaseMs;
+        const abort = (reason: string) => {
+            if (!controller.signal.aborted) {
+                controller.abort(new JobLeaseLostError(reason));
+            }
+        };
+        const lapse = setTimeout(
+            () => abort('region lock lapsed'),
+            Math.max(0, deadline - performance.now()),
+        );
+        const lease: RunLease = {
+            controller,
+            ready: async () => {
+                if (
+                    !controller.signal.aborted &&
+                    deadline - performance.now() >= leaseMs / 2
+                ) {
+                    return true;
+                }
+                abort('too little of the lock left to start');
+                return false;
+            },
+            close: () => {
+                clearTimeout(lapse);
+                this.closeLease(lease);
+            },
+        };
+        this.openLease(lease);
+        return lease;
+    }
+
+    /**
+     * Run a write to the region keys if the gate still holds `token`; -1 if
+     * not.
+     */
+    async #regionWrite(
+        keys: RegionKeys,
+        token: string,
+        op: string,
+        ...args: (string | number)[]
+    ): Promise<number> {
+        const result = await this.clients.redis.eval(
+            REGION_FENCED_WRITE,
+            2,
+            keys.gate,
+            keys.run,
+            token,
+            op,
+            ...args,
+        );
+        return Number(result);
+    }
+
+    // -- Hooks, also for extensions that replace `services.jobs` -------
+
+    /**
+     * Run `fleet` jobs registered here and not running in this process. The
+     * default holds each per region, its lease capped at `every`.
+     */
+    protected async pollFleet(names: string[]): Promise<void> {
+        await Promise.all(
+            names.map((name) => {
+                const job = this.definitionOf(name);
+                return this.track(name, () =>
+                    this.#tryRegion(name, {
+                        ...job,
+                        leaseMs: Math.min(job.leaseMs, job.every),
+                    }),
+                );
+            }),
+        );
+    }
+
+    /** Runs `work` as `name`'s only run in this process. */
+    protected track(name: string, work: () => Promise<void>): Promise<void> {
+        const run = (async () => {
+            try {
+                await work();
+            } catch (err) {
+                console.error(`[jobs] ${name} errored`, err);
+            } finally {
+                this.#running.delete(name);
+            }
+        })();
+        this.#running.set(name, run);
+        return run;
+    }
+
+    protected isRunning(name: string): boolean {
+        return this.#running.has(name);
+    }
+
+    protected definitionOf(name: string): RegisteredJob {
+        const job = this.#jobs.get(name);
+        if (!job) throw new Error(`job not registered: ${name}`);
+        return job;
+    }
+
+    /** Who runs a job here: region, server and process. */
+    protected owner(): string {
+        const region = this.services.broadcast.regionId;
+        const server = this.config.serverId ?? 'local';
+        return `${region}:${server}:${process.pid}`.slice(0, 128);
+    }
+
+    /**
+     * Track `lease` for shutdown; one opened while shutting down is aborted at
+     * once.
+     */
+    protected openLease(lease: RunLease): void {
+        this.#leases.add(lease);
+        if (this.#stopping) {
+            lease.controller.abort(
+                new JobLeaseLostError('server shutting down'),
+            );
+        }
+    }
+
+    /** Stop tracking `lease`; its `close` calls this. */
+    protected closeLease(lease: RunLease): void {
+        this.#leases.delete(lease);
+    }
+
+    protected reportFailure(
+        name: string,
+        failures: number,
+        error: unknown,
+    ): void {
+        console.error(`[jobs] ${name} failed (${failures} in a row)`, error);
+        if (failures < JOB_FAILING_ALARM_AT) return;
+        this.clients.alarm.create(
+            `scheduled_job_failing:${this.namespace}:${name}`,
+            `System job ${name} keeps failing`,
+            {
+                namespace: this.namespace,
+                name,
+                failures,
+                error: asError(error),
+            },
+            'warning',
+            { dedup: true },
+        );
+    }
+}
+
+/**
+ * One hash slot per job. Neither part may contain `:` (`JOB_ID_PATTERN`), so no
+ * two (namespace, name) pairs share keys.
+ */
+const regionKeys = (namespace: string, name: string): RegionKeys => {
+    const tag = `{${namespace}:${name}}`;
+    return {
+        lock: `jobs:${tag}:lock`,
+        gate: `jobs:${tag}:next`,
+        run: `jobs:${tag}:run`,
+    };
+};

src/backend/services/jobs/jobRun.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+// What one job run needs, whichever scheduler holds the job. Also used by
+// extensions that replace `services.jobs`.
+
+import type { JobDefinition } from './JobScheduler.js';
+
+export const JOB_STATE_MAX_BYTES = 64 * 1024;
+export const JOB_ERROR_MAX_CHARS = 1024;
+
+/** The run no longer holds its job; another node may already be running it. */
+export class JobLeaseLostError extends Error {}
+
+export type RegisteredJob = Required<JobDefinition>;
+
+export type RunOutcome = 'success' | 'failure' | 'interrupted';
+
+/** What a run holds the job by. Its deadline is measured from before the claim. */
+export interface RunLease {
+    controller: AbortController;
+    /**
+     * Renews if little lease is left; false (and aborted) if the run can't
+     * start.
+     */
+    ready: () => Promise<boolean>;
+    close: () => void;
+}
+
+export const lostLease = (lease: RunLease, name: string): never => {
+    const lost = new JobLeaseLostError(`job ${name} lost its lease`);
+    lease.controller.abort(lost);
+    throw lost;
+};
+
+/** A run cut short by its signal is interrupted, whether or not it threw. */
+export const runOutcome = (signal: AbortSignal, error: unknown): RunOutcome => {
+    if (signal.aborted) return 'interrupted';
+    return error === null ? 'success' : 'failure';
+};
+
+export const parseState = (raw: string | null): unknown => {
+    if (raw === null) return null;
+    try {
+        return JSON.parse(raw);
+    } catch {
+        return null;
+    }
+};
+
+export const serializeState = (state: unknown): string => {
+    const json = JSON.stringify(state ?? null);
+    if (Buffer.byteLength(json) > JOB_STATE_MAX_BYTES) {
+        throw new Error(`job state is over ${JOB_STATE_MAX_BYTES} bytes`);
+    }
+    return json;
+};
+
+export const asError = (error: unknown): Error =>
+    error instanceof Error ? error : new Error(String(error));
+
+export const errorText = (error: unknown): string =>
+    (error instanceof Error ? error.message : String(error)).slice(
+        0,
+        JOB_ERROR_MAX_CHARS,
+    );

src/backend/services/jobs/jobsTestUtil.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import type { JobRunContext } from './JobScheduler.js';
+
+/**
+ * What a system job's `run` receives, for calling it directly: `saved` records
+ * each `saveState`, and `abort` aborts the signal.
+ */
+export const testJobContext = (
+    state: unknown = null,
+): JobRunContext & { saved: unknown[]; abort: () => void } => {
+    const controller = new AbortController();
+    const saved: unknown[] = [];
+    return {
+        signal: controller.signal,
+        state,
+        saveState: async (next) => {
+            saved.push(next);
+        },
+        saved,
+        abort: () => controller.abort(),
+    };
+};

src/backend/services/jobs/systemJobs.test.ts

@@ -0,0 +1,273 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The backend's own periodic jobs on the default scheduler: what registers,
+ * that each runs once across nodes sharing Redis, and that a single self-hosted
+ * node (sqlite, in-process Redis) runs every one of them.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import {
+    JobScheduler,
+    type JobDefinition,
+    type JobRunContext,
+} from './JobScheduler.js';
+
+type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
+
+const MINUTE = 60_000;
+const HOUR = 60 * MINUTE;
+
+const EXPECTED_JOBS = {
+    'auth.sessions.purge': {
+        every: 7 * 24 * HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 30 * MINUTE,
+    },
+    'kvShare.handles.purgeRevoked': {
+        every: 24 * HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'notifications.retention': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'metering.globalRateCheck': {
+        every: 25 * MINUTE,
+        scope: 'fleet',
+        nearPrimary: false,
+        leaseMs: 5 * MINUTE,
+    },
+    'events.subscriptions.expirySweep': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'events.userHandlers.gc': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'events.subscriptions.creditSweep': {
+        every: 15 * MINUTE,
+        scope: 'fleet',
+        nearPrimary: false,
+        leaseMs: 10 * MINUTE,
+    },
+};
+const JOB_NAMES = Object.keys(EXPECTED_JOBS);
+
+/** A timer armed while the server booted, and the frame that armed it. */
+interface ArmedTimer {
+    kind: 'interval' | 'timeout';
+    ms: number | undefined;
+    caller: string;
+}
+
+let server: PuterServer;
+const registered = new Map<string, JobDefinition>();
+const armed: ArmedTimer[] = [];
+
+/** The first stack frame outside this file and node_modules. */
+const callerOf = (stack: string | undefined): string =>
+    (stack ?? '')
+        .split('\n')
+        .slice(1)
+        .find(
+            (line) =>
+                !line.includes('systemJobs.test') &&
+                !line.includes('node_modules'),
+        ) ?? '';
+
+beforeAll(async () => {
+    const register = vi.spyOn(JobScheduler.prototype, 'register');
+    const realInterval = globalThis.setInterval;
+    const realTimeout = globalThis.setTimeout;
+    const intervals = vi.spyOn(globalThis, 'setInterval').mockImplementation(((
+        ...args: Parameters<typeof setInterval>
+    ) => {
+        armed.push({
+            kind: 'interval',
+            ms: args[1],
+            caller: callerOf(new Error().stack),
+        });
+        return realInterval(...args);
+    }) as never);
+    const timeouts = vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+        ...args: Parameters<typeof setTimeout>
+    ) => {
+        armed.push({
+            kind: 'timeout',
+            ms: args[1],
+            caller: callerOf(new Error().stack),
+        });
+        return realTimeout(...args);
+    }) as never);
+    try {
+        server = await setupTestServer({
+            events: { enabled: true },
+            notificationRetentionDays: 14,
+            maxGlobalUsagePerMinute: 1e15,
+            // A self-hosted node: nothing turns its jobs off.
+            jobs: { enabled: true },
+        } as IConfig);
+    } finally {
+        intervals.mockRestore();
+        timeouts.mockRestore();
+    }
+    for (const [name, definition] of register.mock.calls)
+        registered.set(name, definition);
+    register.mockRestore();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+const keysOf = (namespace: string, name: string) => ({
+    lock: `jobs:{${namespace}:${name}}:lock`,
+    gate: `jobs:{${namespace}:${name}}:next`,
+    run: `jobs:{${namespace}:${name}}:run`,
+});
+
+/** A node of its own in `region`, on the test server's Redis. */
+const node = (namespace: string, region: string): JobScheduler =>
+    new JobScheduler(
+        {
+            domain: 'puter.test',
+            serverId: `node-${region}`,
+            jobs: { namespace },
+        } as IConfig,
+        server.clients as SchedulerArgs[1],
+        server.stores as SchedulerArgs[2],
+        {
+            broadcast: { regionId: region },
+        } as unknown as SchedulerArgs[3],
+    );
+
+describe('system jobs', () => {
+    it('registers every moved job with its real cadence, scope and lease', () => {
+        expect(
+            Object.fromEntries(
+                [...registered].map(([name, job]) => [
+                    name,
+                    {
+                        every: job.every,
+                        scope: job.scope,
+                        nearPrimary: job.nearPrimary ?? false,
+                        leaseMs: job.leaseMs,
+                    },
+                ]),
+            ),
+        ).toEqual(EXPECTED_JOBS);
+    });
+
+    it('leaves no per-node timer behind for a moved job', () => {
+        const armedFrom = (file: string) =>
+            armed
+                .filter(({ caller }) => caller.includes(file))
+                .map(({ kind, ms }) => ({ kind, ms }));
+
+        expect(armedFrom('services/auth/AuthService')).toEqual([]);
+        expect(armedFrom('services/notification/NotificationService')).toEqual(
+            [],
+        );
+        // Node-local: the usage buffer flush.
+        expect(armedFrom('services/metering/MeteringService')).toEqual([
+            { kind: 'interval', ms: 10_000 },
+        ]);
+        // Node-local: the share-job runner (DB claims per job), and its kick.
+        expect(armedFrom('services/kvShare/KvShareService')).toEqual([
+            { kind: 'timeout', ms: expect.any(Number) },
+            { kind: 'interval', ms: MINUTE },
+        ]);
+        // Node-local: the region's pending-delivery sweep.
+        expect(armedFrom('services/events/EventsService')).toEqual([
+            { kind: 'interval', ms: 10_000 },
+        ]);
+    });
+
+    it.each(JOB_NAMES)(
+        'runs %s once across three nodes sharing Redis',
+        async (name) => {
+            const namespace = `ns-${randomUUID()}`;
+            const nodes = ['n1', 'n2', 'n3'].map((id) => node(namespace, id));
+            const job = registered.get(name)!;
+            let runs = 0;
+            for (const scheduler of nodes)
+                scheduler.register(name, {
+                    ...job,
+                    run: async (ctx: JobRunContext) => {
+                        runs += 1;
+                        await job.run(ctx);
+                    },
+                });
+
+            await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+            await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+
+            expect(runs).toBe(1);
+            const redis = server.clients.redis;
+            const keys = keysOf(namespace, name);
+            // A failure would have cut the gate to a 2 s backoff.
+            expect(await redis.pttl(keys.gate)).toBeGreaterThan(
+                job.every - MINUTE,
+            );
+            expect(await redis.hget(keys.run, 'failures')).toBeNull();
+        },
+    );
+
+    it('runs every job on a single self-hosted node, without the old per-job gates', async () => {
+        const jobs = server.services.jobs;
+        const redis = server.clients.redis;
+
+        await jobs.poll();
+
+        // The node's own poll timer may have started some of them first.
+        await vi.waitFor(
+            async () => {
+                for (const name of JOB_NAMES) {
+                    const keys = keysOf(jobs.namespace, name);
+                    expect(await redis.exists(keys.gate), name).toBe(1);
+                    // Released once the run is over.
+                    expect(await redis.exists(keys.lock), name).toBe(0);
+                    expect(await redis.hget(keys.run, 'failures'), name).toBe(
+                        null,
+                    );
+                }
+            },
+            { timeout: 10_000, interval: 100 },
+        );
+        await expect(
+            redis.exists('sessions:purge:gate', 'kvShare:purge:gate'),
+        ).resolves.toBe(0);
+    });
+});

src/backend/services/kvShare/KvShareService.ts

@@ -371,11 +371,9 @@ const SHARE_CHANGE_LOCK_RETRY_MS = 50;
 
 /** Retired handles are kept this long for their owner's listing, then deleted. */
 export const KV_SHARE_REVOKED_RETENTION_SECONDS = 30 * 24 * 60 * 60;
-// Every node checks hourly; the gate lets one node through per day.
-const PURGE_CHECK_MS = 60 * 60 * 1000;
-const PURGE_INITIAL_DELAY_MS = 5 * 60 * 1000;
-const PURGE_GATE_KEY = 'kvShare:purge:gate';
-const PURGE_GATE_SECONDS = 24 * 60 * 60;
+const PURGE_JOB = 'kvShare.handles.purgeRevoked';
+const PURGE_EVERY_MS = 24 * 60 * 60 * 1000;
+const PURGE_LEASE_MS = 15 * 60 * 1000;
 // Rows per delete, with a pause between, so replicas keep up.
 const PURGE_BATCH = 500;
 const PURGE_PAUSE_MS = 50;
@@ -536,7 +534,17 @@ export class KvShareService extends PuterService {
     #jobCursor: string | undefined;
 
     override onServerStart(): void {
-        this.#armPurge();
+        this.services.jobs.register(PURGE_JOB, {
+            every: PURGE_EVERY_MS,
+            scope: 'fleet',
+            nearPrimary: true,
+            leaseMs: PURGE_LEASE_MS,
+            run: async ({ signal }) => {
+                const purged = await this.purgeRevokedHandles(signal);
+                if (purged)
+                    console.log(`[kvShare] purged ${purged} retired handles`);
+            },
+        });
         this.#armJobs();
 
         // A check at one level is answered by a grant at any level above it.
@@ -607,51 +615,24 @@ export class KvShareService extends PuterService {
     // -- Retention purge ---------------------------------------------
 
     /**
-     * Delete handles retired longer ago than the retention, in batches. Returns
-     * the rows deleted, or null when another node holds today's gate. A missing
-     * row reads like a retired one everywhere a handle is resolved.
+     * Delete handles retired longer ago than the retention, in batches, until
+     * none are left or `signal` aborts. Returns the rows deleted. A missing row
+     * reads like a retired one everywhere a handle is resolved.
      */
-    async purgeRevokedHandles(): Promise<number | null> {
-        const claimed = await this.clients.redis.set(
-            PURGE_GATE_KEY,
-            String(Date.now()),
-            'EX',
-            PURGE_GATE_SECONDS,
-            'NX',
-        );
-        if (claimed !== 'OK') return null;
-
+    async purgeRevokedHandles(signal: AbortSignal): Promise<number> {
         const cutoff =
             Math.floor(Date.now() / 1000) - KV_SHARE_REVOKED_RETENTION_SECONDS;
         let purged = 0;
-        let finished = false;
-        try {
-            for (let i = 0; i < PURGE_MAX_BATCHES; i++) {
-                if (this.#stopping) return purged;
-                const batch =
-                    await this.stores.kvShareHandle.deleteRevokedBefore(
-                        cutoff,
-                        PURGE_BATCH,
-                    );
-                purged += batch;
-                if (batch < PURGE_BATCH) break;
-                await new Promise((resolve) =>
-                    setTimeout(resolve, PURGE_PAUSE_MS),
-                );
-            }
-            finished = true;
-            return purged;
-        } finally {
-            // A run cut short frees the gate, so another node picks it up on
-            // its next check instead of tomorrow.
-            if (!finished) {
-                try {
-                    await this.clients.redis.del(PURGE_GATE_KEY);
-                } catch {
-                    // The gate expires on its own.
-                }
-            }
+        for (let i = 0; i < PURGE_MAX_BATCHES && !signal.aborted; i++) {
+            const batch = await this.stores.kvShareHandle.deleteRevokedBefore(
+                cutoff,
+                PURGE_BATCH,
+            );
+            purged += batch;
+            if (batch < PURGE_BATCH) break;
+            await new Promise((resolve) => setTimeout(resolve, PURGE_PAUSE_MS));
         }
+        return purged;
     }
 
     /**
@@ -1047,27 +1028,6 @@ export class KvShareService extends PuterService {
         this.#timers.push(kick, timer);
     }
 
-    #armPurge(): void {
-        const run = async () => {
-            try {
-                const purged = await this.purgeRevokedHandles();
-                if (purged)
-                    console.log(`[kvShare] purged ${purged} retired handles`);
-            } catch (err) {
-                console.warn('[kvShare] retired-handle purge failed', err);
-            }
-        };
-        // Jittered so a deploy does not have every node check at once.
-        const kick = setTimeout(
-            () => void run(),
-            PURGE_INITIAL_DELAY_MS * (0.5 + Math.random()),
-        );
-        kick.unref?.();
-        const timer = setInterval(() => void run(), PURGE_CHECK_MS);
-        timer.unref?.();
-        this.#timers.push(kick, timer);
-    }
-
     /** Whether the level a mint asks for is available on this server. */
     #assertAccessEnabled(access: KvShareAccess): void {
         if (access === 'watch') {
@@ -1745,11 +1705,14 @@ export class KvShareService extends PuterService {
      * The other party of a share, for a notification one party sends through
      * its handle from the app whose data it is. Every check runs before the
      * pair budget is spent. A handle that is unknown, retired, not the
-     * caller's, or seen from another app reads as absent.
+     * caller's, or seen from another app reads as absent. `spendBudget: false`
+     * checks without spending, for a send that is only being scheduled. Called
+     * by extensions.
      */
     async resolveNotifyRecipient(
         actor: Actor,
         handle: unknown,
+        { spendBudget = true }: { spendBudget?: boolean } = {},
     ): Promise<KvShareNotifyTarget> {
         const named = typeof handle === 'string' ? handle : '';
         const senderId = actor.user?.id;
@@ -1782,7 +1745,8 @@ export class KvShareService extends PuterService {
         if (!checked?.holds) throw unknownKvShareHandle(named);
         if (checked.refuses) throw recipientNotAcceptingShares();
 
-        await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
+        if (spendBudget)
+            await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
         return {
             recipientUserId: recipientId,
             appUid: share.appUid,

src/backend/services/localworker/LocalWorkerService.shutdown.test.ts

@@ -0,0 +1,99 @@
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+
+const { disposals } = vi.hoisted(() => ({
+    disposals: [] as Array<ReturnType<typeof vi.fn>>,
+}));
+
+vi.mock('miniflare', () => ({
+    Miniflare: class {
+        dispose = vi.fn(async () => {});
+        constructor() {
+            disposals.push(this.dispose);
+        }
+    },
+}));
+
+describe('LocalWorkerService shutdown disposal', () => {
+    let server: PuterServer;
+    beforeAll(async () => {
+        server = await setupTestServer({
+            api_base_url: 'http://api.puter.localhost',
+        } as never);
+    });
+    afterAll(async () => {
+        await server.shutdown();
+    });
+    afterEach(() => {
+        vi.useRealTimers();
+        disposals.length = 0;
+    });
+
+    it('awaits every disposal even if one fails', async () => {
+        const service = server.services.localworkerservice;
+        await service.cfDeployLocal('first', 'auth', 'export default {}');
+        await service.cfDeployLocal('second', 'auth', 'export default {}');
+        let release!: () => void;
+        const held = new Promise<void>((resolve) => {
+            release = resolve;
+        });
+        disposals[0].mockRejectedValueOnce(new Error('dispose failed'));
+        disposals[1].mockImplementationOnce(async () => {
+            await held;
+        });
+        let finished = false;
+        const draining = Promise.resolve(service.onServerShutdown()).then(
+            () => {
+                finished = true;
+            },
+        );
+        try {
+            await new Promise((resolve) => setTimeout(resolve, 20));
+            expect(
+                disposals.every((dispose) => dispose.mock.calls.length === 1),
+            ).toBe(true);
+            expect(finished).toBe(false);
+        } finally {
+            release();
+            await draining;
+        }
+    });
+
+    it('bounds disposal of a hung worker to five seconds', async () => {
+        const service = server.services.localworkerservice;
+        await service.cfDeployLocal('hung', 'auth', 'export default {}');
+        let release!: () => void;
+        const held = new Promise<void>((resolve) => {
+            release = resolve;
+        });
+        disposals[0].mockImplementationOnce(async () => {
+            await held;
+        });
+        vi.useFakeTimers();
+        try {
+            let finished = false;
+            const draining = Promise.resolve(service.onServerShutdown()).then(
+                () => {
+                    finished = true;
+                },
+            );
+            await vi.advanceTimersByTimeAsync(4_999);
+            expect(finished).toBe(false);
+            await vi.advanceTimersByTimeAsync(1);
+            await draining;
+            expect(vi.getTimerCount()).toBe(0);
+        } finally {
+            release();
+            vi.useRealTimers();
+        }
+    });
+});

src/backend/services/localworker/LocalWorkerService.ts

@@ -10,6 +10,8 @@ import {
 import { puterStores } from '../../stores';
 import type { SubdomainRow } from '../../stores/subdomain/SubdomainStore';
 import { LayerInstances } from '../../types';
+import { settleWithin } from '../../util/concurrency.js';
+import { FINAL_FLUSH_TIMEOUT_MS } from '../../util/gracefulShutdown.js';
 import { PuterService } from '../types';
 
 const MAX_SOURCE_SIZE = 10 * 1024 * 1024; // 10 MB
@@ -240,13 +242,16 @@ export class LocalWorkerService extends PuterService {
         idleSweepTimer.unref?.();
     }
 
-    override onServerShutdown(): void {
+    override async onServerShutdown(): Promise<void> {
         if (idleSweepTimer) {
             clearInterval(idleSweepTimer);
             idleSweepTimer = null;
         }
-        for (const name of [...activeWorkers.keys()]) {
-            void this.#disposeWorker(name);
+        const work = Promise.allSettled(
+            [...activeWorkers.keys()].map((name) => this.#disposeWorker(name)),
+        );
+        if (!(await settleWithin(work, FINAL_FLUSH_TIMEOUT_MS))) {
+            console.warn('[localworker] final flush timed out');
         }
     }
     async reconstructDeployArgs(workerName: string, row: SubdomainRow) {